2024-07-15T06:23:44.623 INFO:root:teuthology version: 0.0.1.dev353+g454a461 2024-07-15T06:23:44.623 DEBUG:teuthology.run:Teuthology command: teuthology --owner scheduled_teuthology@teuthology --archive /home/teuthworker/archive/teuthology-2024-07-10_01:16:02-upgrade:quincy-x-reef-distro-default-smithi/7795172 --description upgrade:quincy-x/stress-split/{0-distro/ubuntu_22.04 0-roles 1-start 2-first-half-tasks/rbd-cls 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/classic} --name teuthology-2024-07-10_01:16:02-upgrade:quincy-x-reef-distro-default-smithi --verbose -- /home/teuthworker/archive/teuthology-2024-07-10_01:16:02-upgrade:quincy-x-reef-distro-default-smithi/7795172/orig.config.yaml 2024-07-15T06:23:44.688 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-07-15T06:23:44.781 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2024-07-10_01:16:02-upgrade:quincy-x-reef-distro-default-smithi/7795172 branch: reef description: upgrade:quincy-x/stress-split/{0-distro/ubuntu_22.04 0-roles 1-start 2-first-half-tasks/rbd-cls 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/classic} email: ceph-qa@ceph.com first-half-sequence: - cephadm.shell: env: - sha1 mon.a: - ceph config set mgr mgr/cephadm/daemon_cache_timeout 60 - ceph config set global log_to_journald false --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps - echo wait for minority of mons to upgrade - while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for majority of mons to upgrade - 'while ! ceph mon versions | grep $sha1 | egrep '': [23]'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for all mons to upgrade - 'while ! ceph mon versions | grep $sha1 | grep '': 3'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for half of osds to upgrade - 'while ! ceph osd versions | grep $sha1 | egrep '': [45678]''; do sleep 2 ; done' - ceph orch upgrade pause - ceph orch ps - ceph orch ps - ceph versions first-half-tasks: - workunit: branch: quincy clients: client.0: - cls/test_cls_rbd.sh - print: '**** done cls/test_cls_rbd.sh 5-workload' first_in_suite: false job_id: '7795172' kernel: kdb: 1 sha1: distro ktype: distro last_in_suite: false machine_type: smithi meta: - desc: 'run basic cls tests for rbd ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run basic cls tests for rbd ' - desc: 'run basic import/export cli tests for rbd ' - desc: 'librbd C and C++ api tests ' - desc: 'randomized correctness test for rados operations on a replicated pool, using only reads, writes, and deletes ' - desc: 'randomized correctness test for rados operations on a replicated pool with snapshot operations ' - desc: 'run basic import/export cli tests for rbd ' name: teuthology-2024-07-10_01:16:02-upgrade:quincy-x-reef-distro-default-smithi no_nested_subset: false openstack: - volumes: count: 4 size: 10 os_type: ubuntu os_version: '22.04' overrides: admin_socket: branch: reef ceph: conf: global: mon election default strategy: 1 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true create_rbd_pool: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) sha1: 6ea7aaa9f93d34dc4d21862afe00140dfddbd706 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: 6ea7aaa9f93d34dc4d21862afe00140dfddbd706 workunit: branch: reef sha1: 9122136bebef94311801796793b3ccd81a4a9765 owner: scheduled_teuthology@teuthology priority: 830 repo: https://git.ceph.com/ceph.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b second-half-sequence: sequential: - cephadm.shell: env: - sha1 mon.a: - ceph orch upgrade resume - sleep 60 - echo wait for upgrade to complete - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done - echo upgrade complete - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 second-half-tasks: - workunit: branch: quincy clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' seed: 2662 sha1: 6ea7aaa9f93d34dc4d21862afe00140dfddbd706 sleep_before_teardown: 0 stress-tasks: - thrashosds: aggressive_pg_num_changes: false chance_force_recovery: 0 chance_pgnum_grow: 1 chance_pgpnum_fix: 1 chance_thrash_cluster_full: 0 chance_thrash_pg_upmap: 0 chance_thrash_pg_upmap_items: 0 disable_objectstore_tool_tests: true timeout: 1200 - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' - workunit: branch: quincy clients: client.0: - cls/test_cls_rbd.sh - print: '**** done cls/test_cls_rbd.sh 5-workload' - workunit: branch: quincy clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' - workunit: branch: quincy clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done rbd/test_librbd.sh 7-workload' - full_sequential: - rados: clients: - client.0 objects: 500 op_weights: delete: 10 read: 45 write: 45 ops: 4000 write_append_excl: false - print: '**** done rados/readwrite 5-workload' - full_sequential: - rados: clients: - client.0 objects: 50 op_weights: delete: 50 read: 100 rollback: 50 snap_create: 50 snap_remove: 50 write: 100 ops: 4000 write_append_excl: false - print: '**** done rados/snaps-few-objects 5-workload' subset: 0/1 suite: upgrade:quincy-x suite_branch: reef suite_path: /home/teuthworker/src/git.ceph.com_ceph_9122136bebef94311801796793b3ccd81a4a9765/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph.git suite_sha1: 9122136bebef94311801796793b3ccd81a4a9765 targets: smithi089.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBJbjO2aBQn2p6AMVpx4RuEghzaDTaewtr6JjGS7B2JZmWbfN2LOwP++abPS3tQd5KjG+ULPdw03xYzkQvVzn8U0= smithi134.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBPEI3iymsZxZYQwBtcj5oyukMHzz1OsDFow47n/X+tCgojrmddpX+9T+8FBNTFojdB5fwgeL3V2HhlrvbSF92Ho= tasks: - install: branch: quincy exclude_packages: - ceph-volume - cephadm: cephadm_branch: quincy cephadm_git_url: https://github.com/ceph/ceph conf: osd: osd_class_default_list: '*' osd_class_load_list: '*' image: quay.ceph.io/ceph-ci/ceph:quincy - cephadm.shell: mon.a: - ceph fs volume create foo - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph.healthy: null - print: '**** upgrading first half of cluster, with stress ****' - parallel: - first-half-tasks - first-half-sequence - print: '**** done upgrading first half of cluster ****' - ceph.healthy: null - print: '**** applying stress + thrashing to mixed-version cluster ****' - parallel: - stress-tasks - ceph.healthy: null - print: '**** finishing upgrade ****' - parallel: - second-half-tasks - second-half-sequence - ceph.healthy: null teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 454a46106b61e201598ae5e1c392af2328539bfe timestamp: 2024-07-10_01:16:02 tube: smithi user: teuthology verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.621193 2024-07-15T06:23:44.781 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph_9122136bebef94311801796793b3ccd81a4a9765/qa; will attempt to use it 2024-07-15T06:23:44.782 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph_9122136bebef94311801796793b3ccd81a4a9765/qa/tasks 2024-07-15T06:23:44.782 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-07-15T06:23:44.784 INFO:teuthology.task.internal:Checking packages... 2024-07-15T06:23:44.804 INFO:teuthology.task.internal:Checking packages for os_type 'ubuntu', flavor 'default' and ceph hash '6ea7aaa9f93d34dc4d21862afe00140dfddbd706' 2024-07-15T06:23:44.804 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-07-15T06:23:44.805 INFO:teuthology.packaging:ref: None 2024-07-15T06:23:44.805 INFO:teuthology.packaging:tag: None 2024-07-15T06:23:44.805 INFO:teuthology.packaging:branch: reef 2024-07-15T06:23:44.805 INFO:teuthology.packaging:sha1: 6ea7aaa9f93d34dc4d21862afe00140dfddbd706 2024-07-15T06:23:44.805 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&ref=reef 2024-07-15T06:23:45.026 INFO:teuthology.task.internal:Found packages for ceph version 18.2.2-1657-gafa9e0a7-1jammy 2024-07-15T06:23:45.027 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-07-15T06:23:45.034 INFO:teuthology.task.internal:no buildpackages task found 2024-07-15T06:23:45.034 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-07-15T06:23:45.049 INFO:teuthology.task.internal:Saving configuration 2024-07-15T06:23:45.072 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-07-15T06:23:45.078 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-07-15T06:23:45.103 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi089.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-07-10_01:16:02-upgrade:quincy-x-reef-distro-default-smithi/7795172', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'ubuntu', 'os_version': '22.04', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-07-15 06:18:04.845468', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMOnWD+cerlVxjGukODual+JQciOHlbQ1bmZu/uHweaM'} 2024-07-15T06:23:45.126 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi134.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-07-10_01:16:02-upgrade:quincy-x-reef-distro-default-smithi/7795172', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'ubuntu', 'os_version': '22.04', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-07-15 06:18:04.847206', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAGn/yH2D/p4U8C3yblRWkeQy/ixQB+Ju0FtFjwv4fbX'} 2024-07-15T06:23:45.126 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-07-15T06:23:45.132 INFO:teuthology.task.internal:roles: ubuntu@smithi089.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2024-07-15T06:23:45.133 INFO:teuthology.task.internal:roles: ubuntu@smithi134.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2024-07-15T06:23:45.133 INFO:teuthology.run_tasks:Running task console_log... 2024-07-15T06:23:45.203 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f27483eeef0>, signals=[15]) 2024-07-15T06:23:45.204 INFO:teuthology.run_tasks:Running task internal.connect... 2024-07-15T06:23:45.210 INFO:teuthology.task.internal:Opening connections... 2024-07-15T06:23:45.210 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi089.front.sepia.ceph.com 2024-07-15T06:23:45.211 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi089.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-07-15T06:23:45.295 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi134.front.sepia.ceph.com 2024-07-15T06:23:45.296 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi134.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-07-15T06:23:45.377 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-07-15T06:23:45.384 DEBUG:teuthology.orchestra.run.smithi089:> uname -m 2024-07-15T06:23:45.389 INFO:teuthology.orchestra.run.smithi089.stdout:x86_64 2024-07-15T06:23:45.390 DEBUG:teuthology.orchestra.run.smithi089:> cat /etc/os-release 2024-07-15T06:23:45.435 INFO:teuthology.orchestra.run.smithi089.stdout:PRETTY_NAME="Ubuntu 22.04.1 LTS" 2024-07-15T06:23:45.435 INFO:teuthology.orchestra.run.smithi089.stdout:NAME="Ubuntu" 2024-07-15T06:23:45.435 INFO:teuthology.orchestra.run.smithi089.stdout:VERSION_ID="22.04" 2024-07-15T06:23:45.435 INFO:teuthology.orchestra.run.smithi089.stdout:VERSION="22.04.1 LTS (Jammy Jellyfish)" 2024-07-15T06:23:45.435 INFO:teuthology.orchestra.run.smithi089.stdout:VERSION_CODENAME=jammy 2024-07-15T06:23:45.435 INFO:teuthology.orchestra.run.smithi089.stdout:ID=ubuntu 2024-07-15T06:23:45.435 INFO:teuthology.orchestra.run.smithi089.stdout:ID_LIKE=debian 2024-07-15T06:23:45.435 INFO:teuthology.orchestra.run.smithi089.stdout:HOME_URL="https://www.ubuntu.com/" 2024-07-15T06:23:45.435 INFO:teuthology.orchestra.run.smithi089.stdout:SUPPORT_URL="https://help.ubuntu.com/" 2024-07-15T06:23:45.435 INFO:teuthology.orchestra.run.smithi089.stdout:BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/" 2024-07-15T06:23:45.435 INFO:teuthology.orchestra.run.smithi089.stdout:PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy" 2024-07-15T06:23:45.435 INFO:teuthology.orchestra.run.smithi089.stdout:UBUNTU_CODENAME=jammy 2024-07-15T06:23:45.436 INFO:teuthology.lock.ops:Updating smithi089.front.sepia.ceph.com on lock server 2024-07-15T06:23:45.454 DEBUG:teuthology.orchestra.run.smithi134:> uname -m 2024-07-15T06:23:45.460 INFO:teuthology.orchestra.run.smithi134.stdout:x86_64 2024-07-15T06:23:45.460 DEBUG:teuthology.orchestra.run.smithi134:> cat /etc/os-release 2024-07-15T06:23:45.507 INFO:teuthology.orchestra.run.smithi134.stdout:PRETTY_NAME="Ubuntu 22.04.1 LTS" 2024-07-15T06:23:45.507 INFO:teuthology.orchestra.run.smithi134.stdout:NAME="Ubuntu" 2024-07-15T06:23:45.507 INFO:teuthology.orchestra.run.smithi134.stdout:VERSION_ID="22.04" 2024-07-15T06:23:45.507 INFO:teuthology.orchestra.run.smithi134.stdout:VERSION="22.04.1 LTS (Jammy Jellyfish)" 2024-07-15T06:23:45.507 INFO:teuthology.orchestra.run.smithi134.stdout:VERSION_CODENAME=jammy 2024-07-15T06:23:45.507 INFO:teuthology.orchestra.run.smithi134.stdout:ID=ubuntu 2024-07-15T06:23:45.507 INFO:teuthology.orchestra.run.smithi134.stdout:ID_LIKE=debian 2024-07-15T06:23:45.507 INFO:teuthology.orchestra.run.smithi134.stdout:HOME_URL="https://www.ubuntu.com/" 2024-07-15T06:23:45.507 INFO:teuthology.orchestra.run.smithi134.stdout:SUPPORT_URL="https://help.ubuntu.com/" 2024-07-15T06:23:45.508 INFO:teuthology.orchestra.run.smithi134.stdout:BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/" 2024-07-15T06:23:45.508 INFO:teuthology.orchestra.run.smithi134.stdout:PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy" 2024-07-15T06:23:45.508 INFO:teuthology.orchestra.run.smithi134.stdout:UBUNTU_CODENAME=jammy 2024-07-15T06:23:45.508 INFO:teuthology.lock.ops:Updating smithi134.front.sepia.ceph.com on lock server 2024-07-15T06:23:45.529 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-07-15T06:23:45.539 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-07-15T06:23:45.544 INFO:teuthology.task.internal:Checking for old test directory... 2024-07-15T06:23:45.544 DEBUG:teuthology.orchestra.run.smithi089:> test '!' -e /home/ubuntu/cephtest 2024-07-15T06:23:45.547 DEBUG:teuthology.orchestra.run.smithi134:> test '!' -e /home/ubuntu/cephtest 2024-07-15T06:23:45.554 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-07-15T06:23:45.560 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-07-15T06:23:45.560 DEBUG:teuthology.orchestra.run.smithi089:> test -z $(ls -A /var/lib/ceph) 2024-07-15T06:23:45.593 DEBUG:teuthology.orchestra.run.smithi134:> test -z $(ls -A /var/lib/ceph) 2024-07-15T06:23:45.685 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-07-15T06:23:45.740 INFO:teuthology.run_tasks:Running task kernel... 2024-07-15T06:23:45.755 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-07-15T06:23:45.755 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': 1, 'sha1': 'distro'}, 'mon.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-07-15T06:23:45.755 DEBUG:teuthology.orchestra.run.smithi089:> test -f /run/.containerenv -o -f /.dockerenv 2024-07-15T06:23:45.756 DEBUG:teuthology.orchestra.run.smithi134:> test -f /run/.containerenv -o -f /.dockerenv 2024-07-15T06:23:45.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-07-15T06:23:45.760 DEBUG:teuthology.orchestra.run.smithi134:> uname -r 2024-07-15T06:23:45.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-07-15T06:23:45.761 DEBUG:teuthology.orchestra.run.smithi089:> uname -r 2024-07-15T06:23:45.810 INFO:teuthology.orchestra.run.smithi134.stdout:5.15.0-56-generic 2024-07-15T06:23:45.810 INFO:teuthology.task.kernel:Running kernel on smithi134: 5.15.0-56-generic 2024-07-15T06:23:45.810 DEBUG:teuthology.orchestra.run.smithi134:> sudo apt-get clean 2024-07-15T06:23:45.812 INFO:teuthology.orchestra.run.smithi089.stdout:5.15.0-56-generic 2024-07-15T06:23:45.812 INFO:teuthology.task.kernel:Running kernel on smithi089: 5.15.0-56-generic 2024-07-15T06:23:45.812 DEBUG:teuthology.orchestra.run.smithi089:> sudo apt-get clean 2024-07-15T06:23:45.936 DEBUG:teuthology.orchestra.run.smithi134:> sudo apt-get update 2024-07-15T06:23:45.941 DEBUG:teuthology.orchestra.run.smithi089:> sudo apt-get update 2024-07-15T06:23:46.069 INFO:teuthology.orchestra.run.smithi134.stdout:Get:1 http://security.ubuntu.com/ubuntu jammy-security InRelease [129 kB] 2024-07-15T06:23:46.073 INFO:teuthology.orchestra.run.smithi089.stdout:Get:1 http://security.ubuntu.com/ubuntu jammy-security InRelease [129 kB] 2024-07-15T06:23:46.181 INFO:teuthology.orchestra.run.smithi089.stdout:Hit:2 http://archive.ubuntu.com/ubuntu jammy InRelease 2024-07-15T06:23:46.186 INFO:teuthology.orchestra.run.smithi134.stdout:Hit:2 http://archive.ubuntu.com/ubuntu jammy InRelease 2024-07-15T06:23:46.262 INFO:teuthology.orchestra.run.smithi089.stdout:Get:3 http://archive.ubuntu.com/ubuntu jammy-updates InRelease [128 kB] 2024-07-15T06:23:46.265 INFO:teuthology.orchestra.run.smithi089.stdout:Get:4 http://security.ubuntu.com/ubuntu jammy-security/main amd64 Packages [1,587 kB] 2024-07-15T06:23:46.272 INFO:teuthology.orchestra.run.smithi134.stdout:Get:3 http://archive.ubuntu.com/ubuntu jammy-updates InRelease [128 kB] 2024-07-15T06:23:46.290 INFO:teuthology.orchestra.run.smithi134.stdout:Get:4 http://security.ubuntu.com/ubuntu jammy-security/main amd64 Packages [1,587 kB] 2024-07-15T06:23:46.369 INFO:teuthology.orchestra.run.smithi089.stdout:Get:5 http://security.ubuntu.com/ubuntu jammy-security/main i386 Packages [498 kB] 2024-07-15T06:23:46.373 INFO:teuthology.orchestra.run.smithi089.stdout:Get:6 http://security.ubuntu.com/ubuntu jammy-security/main Translation-en [269 kB] 2024-07-15T06:23:46.375 INFO:teuthology.orchestra.run.smithi089.stdout:Get:7 http://security.ubuntu.com/ubuntu jammy-security/main amd64 c-n-f Metadata [13.0 kB] 2024-07-15T06:23:46.375 INFO:teuthology.orchestra.run.smithi089.stdout:Get:8 http://security.ubuntu.com/ubuntu jammy-security/restricted i386 Packages [37.3 kB] 2024-07-15T06:23:46.376 INFO:teuthology.orchestra.run.smithi089.stdout:Get:9 http://security.ubuntu.com/ubuntu jammy-security/restricted amd64 Packages [2,037 kB] 2024-07-15T06:23:46.391 INFO:teuthology.orchestra.run.smithi134.stdout:Get:5 http://security.ubuntu.com/ubuntu jammy-security/main i386 Packages [498 kB] 2024-07-15T06:23:46.395 INFO:teuthology.orchestra.run.smithi134.stdout:Get:6 http://security.ubuntu.com/ubuntu jammy-security/main Translation-en [269 kB] 2024-07-15T06:23:46.395 INFO:teuthology.orchestra.run.smithi089.stdout:Get:10 http://security.ubuntu.com/ubuntu jammy-security/restricted Translation-en [348 kB] 2024-07-15T06:23:46.397 INFO:teuthology.orchestra.run.smithi134.stdout:Get:7 http://security.ubuntu.com/ubuntu jammy-security/main amd64 c-n-f Metadata [13.0 kB] 2024-07-15T06:23:46.398 INFO:teuthology.orchestra.run.smithi134.stdout:Get:8 http://security.ubuntu.com/ubuntu jammy-security/restricted i386 Packages [37.3 kB] 2024-07-15T06:23:46.398 INFO:teuthology.orchestra.run.smithi134.stdout:Get:9 http://security.ubuntu.com/ubuntu jammy-security/restricted amd64 Packages [2,037 kB] 2024-07-15T06:23:46.399 INFO:teuthology.orchestra.run.smithi089.stdout:Get:11 http://security.ubuntu.com/ubuntu jammy-security/restricted amd64 c-n-f Metadata [572 B] 2024-07-15T06:23:46.410 INFO:teuthology.orchestra.run.smithi089.stdout:Get:12 http://security.ubuntu.com/ubuntu jammy-security/universe amd64 Packages [884 kB] 2024-07-15T06:23:46.415 INFO:teuthology.orchestra.run.smithi089.stdout:Get:13 http://security.ubuntu.com/ubuntu jammy-security/universe i386 Packages [617 kB] 2024-07-15T06:23:46.417 INFO:teuthology.orchestra.run.smithi134.stdout:Get:10 http://security.ubuntu.com/ubuntu jammy-security/restricted Translation-en [348 kB] 2024-07-15T06:23:46.419 INFO:teuthology.orchestra.run.smithi089.stdout:Get:14 http://security.ubuntu.com/ubuntu jammy-security/universe Translation-en [172 kB] 2024-07-15T06:23:46.420 INFO:teuthology.orchestra.run.smithi134.stdout:Get:11 http://security.ubuntu.com/ubuntu jammy-security/restricted amd64 c-n-f Metadata [572 B] 2024-07-15T06:23:46.420 INFO:teuthology.orchestra.run.smithi134.stdout:Get:12 http://security.ubuntu.com/ubuntu jammy-security/universe i386 Packages [617 kB] 2024-07-15T06:23:46.421 INFO:teuthology.orchestra.run.smithi089.stdout:Get:15 http://security.ubuntu.com/ubuntu jammy-security/universe amd64 c-n-f Metadata [18.9 kB] 2024-07-15T06:23:46.421 INFO:teuthology.orchestra.run.smithi089.stdout:Get:16 http://security.ubuntu.com/ubuntu jammy-security/multiverse i386 Packages [1,356 B] 2024-07-15T06:23:46.421 INFO:teuthology.orchestra.run.smithi089.stdout:Get:17 http://security.ubuntu.com/ubuntu jammy-security/multiverse amd64 Packages [37.2 kB] 2024-07-15T06:23:46.421 INFO:teuthology.orchestra.run.smithi089.stdout:Get:18 http://security.ubuntu.com/ubuntu jammy-security/multiverse Translation-en [7,588 B] 2024-07-15T06:23:46.424 INFO:teuthology.orchestra.run.smithi134.stdout:Get:13 http://security.ubuntu.com/ubuntu jammy-security/universe amd64 Packages [884 kB] 2024-07-15T06:23:46.431 INFO:teuthology.orchestra.run.smithi134.stdout:Get:14 http://security.ubuntu.com/ubuntu jammy-security/universe Translation-en [172 kB] 2024-07-15T06:23:46.432 INFO:teuthology.orchestra.run.smithi134.stdout:Get:15 http://security.ubuntu.com/ubuntu jammy-security/universe amd64 c-n-f Metadata [18.9 kB] 2024-07-15T06:23:46.432 INFO:teuthology.orchestra.run.smithi134.stdout:Get:16 http://security.ubuntu.com/ubuntu jammy-security/multiverse amd64 Packages [37.2 kB] 2024-07-15T06:23:46.433 INFO:teuthology.orchestra.run.smithi134.stdout:Get:17 http://security.ubuntu.com/ubuntu jammy-security/multiverse i386 Packages [1,356 B] 2024-07-15T06:23:46.433 INFO:teuthology.orchestra.run.smithi134.stdout:Get:18 http://security.ubuntu.com/ubuntu jammy-security/multiverse Translation-en [7,588 B] 2024-07-15T06:23:46.444 INFO:teuthology.orchestra.run.smithi089.stdout:Get:19 http://security.ubuntu.com/ubuntu jammy-security/multiverse amd64 c-n-f Metadata [228 B] 2024-07-15T06:23:46.449 INFO:teuthology.orchestra.run.smithi134.stdout:Get:19 http://security.ubuntu.com/ubuntu jammy-security/multiverse amd64 c-n-f Metadata [228 B] 2024-07-15T06:23:46.616 INFO:teuthology.orchestra.run.smithi089.stdout:Get:20 http://archive.ubuntu.com/ubuntu jammy-backports InRelease [127 kB] 2024-07-15T06:23:46.625 INFO:teuthology.orchestra.run.smithi134.stdout:Get:20 http://archive.ubuntu.com/ubuntu jammy-backports InRelease [127 kB] 2024-07-15T06:23:46.753 INFO:teuthology.orchestra.run.smithi089.stdout:Get:21 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 Packages [1,794 kB] 2024-07-15T06:23:46.763 INFO:teuthology.orchestra.run.smithi134.stdout:Get:21 http://archive.ubuntu.com/ubuntu jammy-updates/main i386 Packages [655 kB] 2024-07-15T06:23:46.954 INFO:teuthology.orchestra.run.smithi134.stdout:Get:22 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 Packages [1,794 kB] 2024-07-15T06:23:47.055 INFO:teuthology.orchestra.run.smithi089.stdout:Get:22 http://archive.ubuntu.com/ubuntu jammy-updates/main i386 Packages [655 kB] 2024-07-15T06:23:47.094 INFO:teuthology.orchestra.run.smithi089.stdout:Get:23 http://archive.ubuntu.com/ubuntu jammy-updates/main Translation-en [326 kB] 2024-07-15T06:23:47.106 INFO:teuthology.orchestra.run.smithi134.stdout:Get:23 http://archive.ubuntu.com/ubuntu jammy-updates/main Translation-en [326 kB] 2024-07-15T06:23:47.108 INFO:teuthology.orchestra.run.smithi089.stdout:Get:24 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 c-n-f Metadata [17.7 kB] 2024-07-15T06:23:47.109 INFO:teuthology.orchestra.run.smithi089.stdout:Get:25 http://archive.ubuntu.com/ubuntu jammy-updates/restricted i386 Packages [38.9 kB] 2024-07-15T06:23:47.110 INFO:teuthology.orchestra.run.smithi089.stdout:Get:26 http://archive.ubuntu.com/ubuntu jammy-updates/restricted amd64 Packages [2,093 kB] 2024-07-15T06:23:47.120 INFO:teuthology.orchestra.run.smithi134.stdout:Get:24 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 c-n-f Metadata [17.7 kB] 2024-07-15T06:23:47.121 INFO:teuthology.orchestra.run.smithi134.stdout:Get:25 http://archive.ubuntu.com/ubuntu jammy-updates/restricted amd64 Packages [2,093 kB] 2024-07-15T06:23:47.174 INFO:teuthology.orchestra.run.smithi089.stdout:Get:27 http://archive.ubuntu.com/ubuntu jammy-updates/restricted Translation-en [357 kB] 2024-07-15T06:23:47.182 INFO:teuthology.orchestra.run.smithi089.stdout:Get:28 http://archive.ubuntu.com/ubuntu jammy-updates/restricted amd64 c-n-f Metadata [604 B] 2024-07-15T06:23:47.182 INFO:teuthology.orchestra.run.smithi089.stdout:Get:29 http://archive.ubuntu.com/ubuntu jammy-updates/universe i386 Packages [716 kB] 2024-07-15T06:23:47.185 INFO:teuthology.orchestra.run.smithi134.stdout:Get:26 http://archive.ubuntu.com/ubuntu jammy-updates/restricted i386 Packages [38.9 kB] 2024-07-15T06:23:47.186 INFO:teuthology.orchestra.run.smithi134.stdout:Get:27 http://archive.ubuntu.com/ubuntu jammy-updates/restricted Translation-en [357 kB] 2024-07-15T06:23:47.194 INFO:teuthology.orchestra.run.smithi134.stdout:Get:28 http://archive.ubuntu.com/ubuntu jammy-updates/restricted amd64 c-n-f Metadata [604 B] 2024-07-15T06:23:47.194 INFO:teuthology.orchestra.run.smithi134.stdout:Get:29 http://archive.ubuntu.com/ubuntu jammy-updates/universe i386 Packages [716 kB] 2024-07-15T06:23:47.197 INFO:teuthology.orchestra.run.smithi089.stdout:Get:30 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 Packages [1,101 kB] 2024-07-15T06:23:47.209 INFO:teuthology.orchestra.run.smithi134.stdout:Get:30 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 Packages [1,101 kB] 2024-07-15T06:23:47.233 INFO:teuthology.orchestra.run.smithi089.stdout:Get:31 http://archive.ubuntu.com/ubuntu jammy-updates/universe Translation-en [256 kB] 2024-07-15T06:23:47.237 INFO:teuthology.orchestra.run.smithi089.stdout:Get:32 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 c-n-f Metadata [25.4 kB] 2024-07-15T06:23:47.238 INFO:teuthology.orchestra.run.smithi089.stdout:Get:33 http://archive.ubuntu.com/ubuntu jammy-updates/multiverse amd64 Packages [43.3 kB] 2024-07-15T06:23:47.239 INFO:teuthology.orchestra.run.smithi089.stdout:Get:34 http://archive.ubuntu.com/ubuntu jammy-updates/multiverse i386 Packages [4,744 B] 2024-07-15T06:23:47.240 INFO:teuthology.orchestra.run.smithi089.stdout:Get:35 http://archive.ubuntu.com/ubuntu jammy-updates/multiverse Translation-en [10.8 kB] 2024-07-15T06:23:47.244 INFO:teuthology.orchestra.run.smithi134.stdout:Get:31 http://archive.ubuntu.com/ubuntu jammy-updates/universe Translation-en [256 kB] 2024-07-15T06:23:47.249 INFO:teuthology.orchestra.run.smithi134.stdout:Get:32 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 c-n-f Metadata [25.4 kB] 2024-07-15T06:23:47.271 INFO:teuthology.orchestra.run.smithi134.stdout:Get:33 http://archive.ubuntu.com/ubuntu jammy-updates/multiverse amd64 Packages [43.3 kB] 2024-07-15T06:23:47.272 INFO:teuthology.orchestra.run.smithi134.stdout:Get:34 http://archive.ubuntu.com/ubuntu jammy-updates/multiverse i386 Packages [4,744 B] 2024-07-15T06:23:47.272 INFO:teuthology.orchestra.run.smithi134.stdout:Get:35 http://archive.ubuntu.com/ubuntu jammy-updates/multiverse Translation-en [10.8 kB] 2024-07-15T06:23:47.301 INFO:teuthology.orchestra.run.smithi089.stdout:Get:36 http://archive.ubuntu.com/ubuntu jammy-updates/multiverse amd64 c-n-f Metadata [444 B] 2024-07-15T06:23:47.301 INFO:teuthology.orchestra.run.smithi089.stdout:Get:37 http://archive.ubuntu.com/ubuntu jammy-backports/main amd64 Packages [67.1 kB] 2024-07-15T06:23:47.302 INFO:teuthology.orchestra.run.smithi089.stdout:Get:38 http://archive.ubuntu.com/ubuntu jammy-backports/main i386 Packages [59.3 kB] 2024-07-15T06:23:47.303 INFO:teuthology.orchestra.run.smithi089.stdout:Get:39 http://archive.ubuntu.com/ubuntu jammy-backports/main Translation-en [11.0 kB] 2024-07-15T06:23:47.332 INFO:teuthology.orchestra.run.smithi134.stdout:Get:36 http://archive.ubuntu.com/ubuntu jammy-updates/multiverse amd64 c-n-f Metadata [444 B] 2024-07-15T06:23:47.332 INFO:teuthology.orchestra.run.smithi134.stdout:Get:37 http://archive.ubuntu.com/ubuntu jammy-backports/main i386 Packages [59.3 kB] 2024-07-15T06:23:47.333 INFO:teuthology.orchestra.run.smithi134.stdout:Get:38 http://archive.ubuntu.com/ubuntu jammy-backports/main amd64 Packages [67.1 kB] 2024-07-15T06:23:47.334 INFO:teuthology.orchestra.run.smithi134.stdout:Get:39 http://archive.ubuntu.com/ubuntu jammy-backports/main Translation-en [11.0 kB] 2024-07-15T06:23:47.335 INFO:teuthology.orchestra.run.smithi134.stdout:Get:40 http://archive.ubuntu.com/ubuntu jammy-backports/main amd64 c-n-f Metadata [388 B] 2024-07-15T06:23:47.353 INFO:teuthology.orchestra.run.smithi134.stdout:Get:41 http://archive.ubuntu.com/ubuntu jammy-backports/universe i386 Packages [16.0 kB] 2024-07-15T06:23:47.353 INFO:teuthology.orchestra.run.smithi134.stdout:Get:42 http://archive.ubuntu.com/ubuntu jammy-backports/universe amd64 Packages [27.6 kB] 2024-07-15T06:23:47.382 INFO:teuthology.orchestra.run.smithi089.stdout:Get:40 http://archive.ubuntu.com/ubuntu jammy-backports/main amd64 c-n-f Metadata [388 B] 2024-07-15T06:23:47.382 INFO:teuthology.orchestra.run.smithi089.stdout:Get:41 http://archive.ubuntu.com/ubuntu jammy-backports/universe i386 Packages [16.0 kB] 2024-07-15T06:23:47.383 INFO:teuthology.orchestra.run.smithi089.stdout:Get:42 http://archive.ubuntu.com/ubuntu jammy-backports/universe amd64 Packages [27.6 kB] 2024-07-15T06:23:47.384 INFO:teuthology.orchestra.run.smithi089.stdout:Get:43 http://archive.ubuntu.com/ubuntu jammy-backports/universe Translation-en [16.5 kB] 2024-07-15T06:23:47.384 INFO:teuthology.orchestra.run.smithi089.stdout:Get:44 http://archive.ubuntu.com/ubuntu jammy-backports/universe amd64 c-n-f Metadata [672 B] 2024-07-15T06:23:47.434 INFO:teuthology.orchestra.run.smithi134.stdout:Get:43 http://archive.ubuntu.com/ubuntu jammy-backports/universe Translation-en [16.5 kB] 2024-07-15T06:23:47.434 INFO:teuthology.orchestra.run.smithi134.stdout:Get:44 http://archive.ubuntu.com/ubuntu jammy-backports/universe amd64 c-n-f Metadata [672 B] 2024-07-15T06:23:52.442 INFO:teuthology.orchestra.run.smithi089.stdout:Fetched 14.6 MB in 2s (7,416 kB/s) 2024-07-15T06:23:52.446 INFO:teuthology.orchestra.run.smithi134.stdout:Fetched 14.6 MB in 2s (7,308 kB/s) 2024-07-15T06:23:53.734 INFO:teuthology.orchestra.run.smithi089.stdout:Reading package lists... 2024-07-15T06:23:53.752 DEBUG:teuthology.orchestra.run.smithi089:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic 2024-07-15T06:23:53.814 INFO:teuthology.orchestra.run.smithi089.stdout:Reading package lists... 2024-07-15T06:23:53.847 INFO:teuthology.orchestra.run.smithi134.stdout:Reading package lists... 2024-07-15T06:23:53.865 DEBUG:teuthology.orchestra.run.smithi134:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic 2024-07-15T06:23:53.926 INFO:teuthology.orchestra.run.smithi134.stdout:Reading package lists... 2024-07-15T06:23:54.018 INFO:teuthology.orchestra.run.smithi089.stdout:Building dependency tree... 2024-07-15T06:23:54.018 INFO:teuthology.orchestra.run.smithi089.stdout:Reading state information... 2024-07-15T06:23:54.129 INFO:teuthology.orchestra.run.smithi134.stdout:Building dependency tree... 2024-07-15T06:23:54.130 INFO:teuthology.orchestra.run.smithi134.stdout:Reading state information... 2024-07-15T06:23:54.201 INFO:teuthology.orchestra.run.smithi089.stdout:The following packages were automatically installed and are no longer required: 2024-07-15T06:23:54.202 INFO:teuthology.orchestra.run.smithi089.stdout: libboost-iostreams1.74.0 libboost-thread1.74.0 libgfapi0 libgfrpc0 libgfxdr0 2024-07-15T06:23:54.202 INFO:teuthology.orchestra.run.smithi089.stdout: libglusterfs0 libiscsi7 libpmemobj1 libpython2-dev libpython2-stdlib 2024-07-15T06:23:54.202 INFO:teuthology.orchestra.run.smithi089.stdout: libpython2.7 libpython2.7-dev libpython2.7-minimal libpython2.7-stdlib 2024-07-15T06:23:54.202 INFO:teuthology.orchestra.run.smithi089.stdout: python2 python2-dev python2-minimal python2.7 python2.7-dev 2024-07-15T06:23:54.202 INFO:teuthology.orchestra.run.smithi089.stdout: python2.7-minimal 2024-07-15T06:23:54.202 INFO:teuthology.orchestra.run.smithi089.stdout:Use 'sudo apt autoremove' to remove them. 2024-07-15T06:23:54.204 INFO:teuthology.orchestra.run.smithi089.stdout:The following additional packages will be installed: 2024-07-15T06:23:54.204 INFO:teuthology.orchestra.run.smithi089.stdout: linux-generic linux-headers-5.15.0-113 linux-headers-5.15.0-113-generic 2024-07-15T06:23:54.204 INFO:teuthology.orchestra.run.smithi089.stdout: linux-headers-generic linux-image-5.15.0-113-generic 2024-07-15T06:23:54.204 INFO:teuthology.orchestra.run.smithi089.stdout: linux-modules-5.15.0-113-generic linux-modules-extra-5.15.0-113-generic 2024-07-15T06:23:54.205 INFO:teuthology.orchestra.run.smithi089.stdout:Suggested packages: 2024-07-15T06:23:54.205 INFO:teuthology.orchestra.run.smithi089.stdout: fdutils linux-doc | linux-source-5.15.0 linux-tools 2024-07-15T06:23:54.206 INFO:teuthology.orchestra.run.smithi089.stdout:Recommended packages: 2024-07-15T06:23:54.206 INFO:teuthology.orchestra.run.smithi089.stdout: thermald 2024-07-15T06:23:54.257 INFO:teuthology.orchestra.run.smithi089.stdout:The following NEW packages will be installed: 2024-07-15T06:23:54.258 INFO:teuthology.orchestra.run.smithi089.stdout: linux-headers-5.15.0-113 linux-headers-5.15.0-113-generic 2024-07-15T06:23:54.258 INFO:teuthology.orchestra.run.smithi089.stdout: linux-image-5.15.0-113-generic linux-modules-5.15.0-113-generic 2024-07-15T06:23:54.258 INFO:teuthology.orchestra.run.smithi089.stdout: linux-modules-extra-5.15.0-113-generic 2024-07-15T06:23:54.259 INFO:teuthology.orchestra.run.smithi089.stdout:The following packages will be upgraded: 2024-07-15T06:23:54.260 INFO:teuthology.orchestra.run.smithi089.stdout: linux-generic linux-headers-generic linux-image-generic 2024-07-15T06:23:54.302 INFO:teuthology.orchestra.run.smithi134.stdout:The following packages were automatically installed and are no longer required: 2024-07-15T06:23:54.302 INFO:teuthology.orchestra.run.smithi134.stdout: libboost-iostreams1.74.0 libboost-thread1.74.0 libgfapi0 libgfrpc0 libgfxdr0 2024-07-15T06:23:54.302 INFO:teuthology.orchestra.run.smithi134.stdout: libglusterfs0 libiscsi7 libpmemobj1 libpython2-dev libpython2-stdlib 2024-07-15T06:23:54.303 INFO:teuthology.orchestra.run.smithi134.stdout: libpython2.7 libpython2.7-dev libpython2.7-minimal libpython2.7-stdlib 2024-07-15T06:23:54.303 INFO:teuthology.orchestra.run.smithi134.stdout: python2 python2-dev python2-minimal python2.7 python2.7-dev 2024-07-15T06:23:54.303 INFO:teuthology.orchestra.run.smithi134.stdout: python2.7-minimal 2024-07-15T06:23:54.303 INFO:teuthology.orchestra.run.smithi134.stdout:Use 'sudo apt autoremove' to remove them. 2024-07-15T06:23:54.304 INFO:teuthology.orchestra.run.smithi134.stdout:The following additional packages will be installed: 2024-07-15T06:23:54.304 INFO:teuthology.orchestra.run.smithi134.stdout: linux-generic linux-headers-5.15.0-113 linux-headers-5.15.0-113-generic 2024-07-15T06:23:54.304 INFO:teuthology.orchestra.run.smithi134.stdout: linux-headers-generic linux-image-5.15.0-113-generic 2024-07-15T06:23:54.305 INFO:teuthology.orchestra.run.smithi134.stdout: linux-modules-5.15.0-113-generic linux-modules-extra-5.15.0-113-generic 2024-07-15T06:23:54.306 INFO:teuthology.orchestra.run.smithi134.stdout:Suggested packages: 2024-07-15T06:23:54.306 INFO:teuthology.orchestra.run.smithi134.stdout: fdutils linux-doc | linux-source-5.15.0 linux-tools 2024-07-15T06:23:54.306 INFO:teuthology.orchestra.run.smithi134.stdout:Recommended packages: 2024-07-15T06:23:54.306 INFO:teuthology.orchestra.run.smithi134.stdout: thermald 2024-07-15T06:23:54.359 INFO:teuthology.orchestra.run.smithi134.stdout:The following NEW packages will be installed: 2024-07-15T06:23:54.359 INFO:teuthology.orchestra.run.smithi134.stdout: linux-headers-5.15.0-113 linux-headers-5.15.0-113-generic 2024-07-15T06:23:54.359 INFO:teuthology.orchestra.run.smithi134.stdout: linux-image-5.15.0-113-generic linux-modules-5.15.0-113-generic 2024-07-15T06:23:54.359 INFO:teuthology.orchestra.run.smithi134.stdout: linux-modules-extra-5.15.0-113-generic 2024-07-15T06:23:54.360 INFO:teuthology.orchestra.run.smithi134.stdout:The following packages will be upgraded: 2024-07-15T06:23:54.361 INFO:teuthology.orchestra.run.smithi134.stdout: linux-generic linux-headers-generic linux-image-generic 2024-07-15T06:23:54.439 INFO:teuthology.orchestra.run.smithi089.stdout:3 upgraded, 5 newly installed, 0 to remove and 329 not upgraded. 2024-07-15T06:23:54.439 INFO:teuthology.orchestra.run.smithi089.stdout:Need to get 113 MB of archives. 2024-07-15T06:23:54.439 INFO:teuthology.orchestra.run.smithi089.stdout:After this operation, 583 MB of additional disk space will be used. 2024-07-15T06:23:54.439 INFO:teuthology.orchestra.run.smithi089.stdout:Get:1 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-modules-5.15.0-113-generic amd64 5.15.0-113.123 [22.6 MB] 2024-07-15T06:23:54.541 INFO:teuthology.orchestra.run.smithi134.stdout:3 upgraded, 5 newly installed, 0 to remove and 329 not upgraded. 2024-07-15T06:23:54.541 INFO:teuthology.orchestra.run.smithi134.stdout:Need to get 113 MB of archives. 2024-07-15T06:23:54.541 INFO:teuthology.orchestra.run.smithi134.stdout:After this operation, 583 MB of additional disk space will be used. 2024-07-15T06:23:54.541 INFO:teuthology.orchestra.run.smithi134.stdout:Get:1 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-modules-5.15.0-113-generic amd64 5.15.0-113.123 [22.6 MB] 2024-07-15T06:23:55.916 INFO:teuthology.orchestra.run.smithi089.stdout:Get:2 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-image-5.15.0-113-generic amd64 5.15.0-113.123 [11.5 MB] 2024-07-15T06:23:55.933 INFO:teuthology.orchestra.run.smithi134.stdout:Get:2 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-image-5.15.0-113-generic amd64 5.15.0-113.123 [11.5 MB] 2024-07-15T06:23:56.261 INFO:teuthology.orchestra.run.smithi089.stdout:Get:3 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-modules-extra-5.15.0-113-generic amd64 5.15.0-113.123 [63.9 MB] 2024-07-15T06:23:56.317 INFO:teuthology.orchestra.run.smithi134.stdout:Get:3 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-modules-extra-5.15.0-113-generic amd64 5.15.0-113.123 [63.9 MB] 2024-07-15T06:23:58.490 INFO:teuthology.orchestra.run.smithi089.stdout:Get:4 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-generic amd64 5.15.0.113.113 [1,692 B] 2024-07-15T06:23:58.490 INFO:teuthology.orchestra.run.smithi089.stdout:Get:5 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-image-generic amd64 5.15.0.113.113 [2,488 B] 2024-07-15T06:23:58.491 INFO:teuthology.orchestra.run.smithi089.stdout:Get:6 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-headers-5.15.0-113 all 5.15.0-113.123 [12.3 MB] 2024-07-15T06:23:58.505 INFO:teuthology.orchestra.run.smithi134.stdout:Get:4 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-generic amd64 5.15.0.113.113 [1,692 B] 2024-07-15T06:23:58.505 INFO:teuthology.orchestra.run.smithi134.stdout:Get:5 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-image-generic amd64 5.15.0.113.113 [2,488 B] 2024-07-15T06:23:58.506 INFO:teuthology.orchestra.run.smithi134.stdout:Get:6 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-headers-5.15.0-113 all 5.15.0-113.123 [12.3 MB] 2024-07-15T06:23:59.195 INFO:teuthology.orchestra.run.smithi089.stdout:Get:7 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-headers-5.15.0-113-generic amd64 5.15.0-113.123 [2,895 kB] 2024-07-15T06:23:59.312 INFO:teuthology.orchestra.run.smithi089.stdout:Get:8 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-headers-generic amd64 5.15.0.113.113 [2,346 B] 2024-07-15T06:23:59.373 INFO:teuthology.orchestra.run.smithi134.stdout:Get:7 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-headers-5.15.0-113-generic amd64 5.15.0-113.123 [2,895 kB] 2024-07-15T06:23:59.504 INFO:teuthology.orchestra.run.smithi134.stdout:Get:8 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-headers-generic amd64 5.15.0.113.113 [2,346 B] 2024-07-15T06:23:59.732 INFO:teuthology.orchestra.run.smithi089.stdout:Fetched 113 MB in 5s (22.5 MB/s) 2024-07-15T06:23:59.891 INFO:teuthology.orchestra.run.smithi134.stdout:Fetched 113 MB in 5s (22.1 MB/s) 2024-07-15T06:23:59.920 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package linux-modules-5.15.0-113-generic. 2024-07-15T06:24:00.076 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package linux-modules-5.15.0-113-generic. 2024-07-15T06:24:01.881 INFO:teuthology.orchestra.run.smithi089.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 129542 files and directories currently installed.) 2024-07-15T06:24:01.885 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../0-linux-modules-5.15.0-113-generic_5.15.0-113.123_amd64.deb ... 2024-07-15T06:24:02.011 INFO:teuthology.orchestra.run.smithi134.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 129542 files and directories currently installed.) 2024-07-15T06:24:02.015 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../0-linux-modules-5.15.0-113-generic_5.15.0-113.123_amd64.deb ... 2024-07-15T06:24:02.051 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking linux-modules-5.15.0-113-generic (5.15.0-113.123) ... 2024-07-15T06:24:02.122 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking linux-modules-5.15.0-113-generic (5.15.0-113.123) ... 2024-07-15T06:24:04.461 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package linux-image-5.15.0-113-generic. 2024-07-15T06:24:04.482 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../1-linux-image-5.15.0-113-generic_5.15.0-113.123_amd64.deb ... 2024-07-15T06:24:04.565 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking linux-image-5.15.0-113-generic (5.15.0-113.123) ... 2024-07-15T06:24:04.764 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package linux-image-5.15.0-113-generic. 2024-07-15T06:24:04.785 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../1-linux-image-5.15.0-113-generic_5.15.0-113.123_amd64.deb ... 2024-07-15T06:24:04.868 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking linux-image-5.15.0-113-generic (5.15.0-113.123) ... 2024-07-15T06:24:05.039 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package linux-modules-extra-5.15.0-113-generic. 2024-07-15T06:24:05.059 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../2-linux-modules-extra-5.15.0-113-generic_5.15.0-113.123_amd64.deb ... 2024-07-15T06:24:05.089 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking linux-modules-extra-5.15.0-113-generic (5.15.0-113.123) ... 2024-07-15T06:24:05.384 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package linux-modules-extra-5.15.0-113-generic. 2024-07-15T06:24:05.404 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../2-linux-modules-extra-5.15.0-113-generic_5.15.0-113.123_amd64.deb ... 2024-07-15T06:24:05.442 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking linux-modules-extra-5.15.0-113-generic (5.15.0-113.123) ... 2024-07-15T06:24:11.473 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../3-linux-generic_5.15.0.113.113_amd64.deb ... 2024-07-15T06:24:11.574 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking linux-generic (5.15.0.113.113) over (5.15.0.56.54) ... 2024-07-15T06:24:11.924 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../4-linux-image-generic_5.15.0.113.113_amd64.deb ... 2024-07-15T06:24:12.045 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking linux-image-generic (5.15.0.113.113) over (5.15.0.56.54) ... 2024-07-15T06:24:12.135 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../3-linux-generic_5.15.0.113.113_amd64.deb ... 2024-07-15T06:24:12.243 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking linux-generic (5.15.0.113.113) over (5.15.0.56.54) ... 2024-07-15T06:24:12.314 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package linux-headers-5.15.0-113. 2024-07-15T06:24:12.337 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../5-linux-headers-5.15.0-113_5.15.0-113.123_all.deb ... 2024-07-15T06:24:12.381 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking linux-headers-5.15.0-113 (5.15.0-113.123) ... 2024-07-15T06:24:12.601 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../4-linux-image-generic_5.15.0.113.113_amd64.deb ... 2024-07-15T06:24:12.729 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking linux-image-generic (5.15.0.113.113) over (5.15.0.56.54) ... 2024-07-15T06:24:13.057 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package linux-headers-5.15.0-113. 2024-07-15T06:24:13.079 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../5-linux-headers-5.15.0-113_5.15.0-113.123_all.deb ... 2024-07-15T06:24:13.115 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking linux-headers-5.15.0-113 (5.15.0-113.123) ... 2024-07-15T06:24:17.780 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package linux-headers-5.15.0-113-generic. 2024-07-15T06:24:17.817 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../6-linux-headers-5.15.0-113-generic_5.15.0-113.123_amd64.deb ... 2024-07-15T06:24:17.855 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking linux-headers-5.15.0-113-generic (5.15.0-113.123) ... 2024-07-15T06:24:19.489 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package linux-headers-5.15.0-113-generic. 2024-07-15T06:24:19.526 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../6-linux-headers-5.15.0-113-generic_5.15.0-113.123_amd64.deb ... 2024-07-15T06:24:19.563 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking linux-headers-5.15.0-113-generic (5.15.0-113.123) ... 2024-07-15T06:24:20.947 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../7-linux-headers-generic_5.15.0.113.113_amd64.deb ... 2024-07-15T06:24:21.061 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking linux-headers-generic (5.15.0.113.113) over (5.15.0.56.54) ... 2024-07-15T06:24:21.394 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up linux-headers-5.15.0-113 (5.15.0-113.123) ... 2024-07-15T06:24:21.958 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up linux-headers-5.15.0-113-generic (5.15.0-113.123) ... 2024-07-15T06:24:22.426 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up linux-headers-generic (5.15.0.113.113) ... 2024-07-15T06:24:22.554 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up linux-modules-5.15.0-113-generic (5.15.0-113.123) ... 2024-07-15T06:24:23.385 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../7-linux-headers-generic_5.15.0.113.113_amd64.deb ... 2024-07-15T06:24:23.526 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking linux-headers-generic (5.15.0.113.113) over (5.15.0.56.54) ... 2024-07-15T06:24:23.868 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up linux-headers-5.15.0-113 (5.15.0-113.123) ... 2024-07-15T06:24:23.958 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up linux-image-5.15.0-113-generic (5.15.0-113.123) ... 2024-07-15T06:24:24.006 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up linux-headers-5.15.0-113-generic (5.15.0-113.123) ... 2024-07-15T06:24:24.115 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up linux-headers-generic (5.15.0.113.113) ... 2024-07-15T06:24:24.227 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up linux-modules-5.15.0-113-generic (5.15.0-113.123) ... 2024-07-15T06:24:25.317 INFO:teuthology.orchestra.run.smithi089.stdout:I: /vmlinuz.old is now a symlink to boot/vmlinuz-5.15.0-56-generic 2024-07-15T06:24:25.317 INFO:teuthology.orchestra.run.smithi089.stdout:I: /initrd.img.old is now a symlink to boot/initrd.img-5.15.0-56-generic 2024-07-15T06:24:25.317 INFO:teuthology.orchestra.run.smithi089.stdout:I: /vmlinuz is now a symlink to boot/vmlinuz-5.15.0-113-generic 2024-07-15T06:24:25.317 INFO:teuthology.orchestra.run.smithi089.stdout:I: /initrd.img is now a symlink to boot/initrd.img-5.15.0-113-generic 2024-07-15T06:24:25.434 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up linux-modules-extra-5.15.0-113-generic (5.15.0-113.123) ... 2024-07-15T06:24:25.630 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up linux-image-5.15.0-113-generic (5.15.0-113.123) ... 2024-07-15T06:24:26.811 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up linux-image-generic (5.15.0.113.113) ... 2024-07-15T06:24:26.937 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up linux-generic (5.15.0.113.113) ... 2024-07-15T06:24:26.996 INFO:teuthology.orchestra.run.smithi134.stdout:I: /vmlinuz.old is now a symlink to boot/vmlinuz-5.15.0-56-generic 2024-07-15T06:24:26.996 INFO:teuthology.orchestra.run.smithi134.stdout:I: /initrd.img.old is now a symlink to boot/initrd.img-5.15.0-56-generic 2024-07-15T06:24:26.996 INFO:teuthology.orchestra.run.smithi134.stdout:I: /vmlinuz is now a symlink to boot/vmlinuz-5.15.0-113-generic 2024-07-15T06:24:26.996 INFO:teuthology.orchestra.run.smithi134.stdout:I: /initrd.img is now a symlink to boot/initrd.img-5.15.0-113-generic 2024-07-15T06:24:27.063 INFO:teuthology.orchestra.run.smithi089.stdout:Processing triggers for linux-image-5.15.0-113-generic (5.15.0-113.123) ... 2024-07-15T06:24:27.134 INFO:teuthology.orchestra.run.smithi089.stdout:/etc/kernel/postinst.d/initramfs-tools: 2024-07-15T06:24:27.134 INFO:teuthology.orchestra.run.smithi089.stdout:update-initramfs: Generating /boot/initrd.img-5.15.0-113-generic 2024-07-15T06:24:27.157 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up linux-modules-extra-5.15.0-113-generic (5.15.0-113.123) ... 2024-07-15T06:24:28.766 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up linux-image-generic (5.15.0.113.113) ... 2024-07-15T06:24:28.901 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up linux-generic (5.15.0.113.113) ... 2024-07-15T06:24:29.018 INFO:teuthology.orchestra.run.smithi134.stdout:Processing triggers for linux-image-5.15.0-113-generic (5.15.0-113.123) ... 2024-07-15T06:24:29.077 INFO:teuthology.orchestra.run.smithi134.stdout:/etc/kernel/postinst.d/initramfs-tools: 2024-07-15T06:24:29.077 INFO:teuthology.orchestra.run.smithi134.stdout:update-initramfs: Generating /boot/initrd.img-5.15.0-113-generic 2024-07-15T06:24:41.235 INFO:teuthology.orchestra.run.smithi089.stdout:/etc/kernel/postinst.d/zz-update-grub: 2024-07-15T06:24:41.235 INFO:teuthology.orchestra.run.smithi089.stdout:Sourcing file `/etc/default/grub' 2024-07-15T06:24:41.251 INFO:teuthology.orchestra.run.smithi089.stdout:Sourcing file `/etc/default/grub.d/init-select.cfg' 2024-07-15T06:24:41.268 INFO:teuthology.orchestra.run.smithi089.stdout:Generating grub configuration file ... 2024-07-15T06:24:41.563 INFO:teuthology.orchestra.run.smithi089.stdout:Found linux image: /boot/vmlinuz-5.15.0-113-generic 2024-07-15T06:24:41.582 INFO:teuthology.orchestra.run.smithi089.stdout:Found initrd image: /boot/initrd.img-5.15.0-113-generic 2024-07-15T06:24:41.884 INFO:teuthology.orchestra.run.smithi089.stdout:Found linux image: /boot/vmlinuz-5.15.0-56-generic 2024-07-15T06:24:41.890 INFO:teuthology.orchestra.run.smithi089.stdout:Found initrd image: /boot/initrd.img-5.15.0-56-generic 2024-07-15T06:24:41.962 INFO:teuthology.orchestra.run.smithi089.stdout:Found linux image: /boot/vmlinuz-5.4.0-135-generic 2024-07-15T06:24:41.968 INFO:teuthology.orchestra.run.smithi089.stdout:Found initrd image: /boot/initrd.img-5.4.0-135-generic 2024-07-15T06:24:42.139 INFO:teuthology.orchestra.run.smithi089.stdout:Warning: os-prober will not be executed to detect other bootable partitions. 2024-07-15T06:24:42.140 INFO:teuthology.orchestra.run.smithi089.stdout:Systems on them will not be added to the GRUB boot configuration. 2024-07-15T06:24:42.140 INFO:teuthology.orchestra.run.smithi089.stdout:Check GRUB_DISABLE_OS_PROBER documentation entry. 2024-07-15T06:24:42.180 INFO:teuthology.orchestra.run.smithi089.stdout:done 2024-07-15T06:24:42.651 DEBUG:teuthology.orchestra.run.smithi089:> dpkg -s linux-image-generic 2024-07-15T06:24:42.676 INFO:teuthology.orchestra.run.smithi089.stdout:Package: linux-image-generic 2024-07-15T06:24:42.676 INFO:teuthology.orchestra.run.smithi089.stdout:Status: install ok installed 2024-07-15T06:24:42.676 INFO:teuthology.orchestra.run.smithi089.stdout:Priority: optional 2024-07-15T06:24:42.676 INFO:teuthology.orchestra.run.smithi089.stdout:Section: kernel 2024-07-15T06:24:42.676 INFO:teuthology.orchestra.run.smithi089.stdout:Installed-Size: 21 2024-07-15T06:24:42.676 INFO:teuthology.orchestra.run.smithi089.stdout:Maintainer: Ubuntu Kernel Team 2024-07-15T06:24:42.677 INFO:teuthology.orchestra.run.smithi089.stdout:Architecture: amd64 2024-07-15T06:24:42.677 INFO:teuthology.orchestra.run.smithi089.stdout:Source: linux-meta 2024-07-15T06:24:42.677 INFO:teuthology.orchestra.run.smithi089.stdout:Version: 5.15.0.113.113 2024-07-15T06:24:42.677 INFO:teuthology.orchestra.run.smithi089.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.3), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-113), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.3) 2024-07-15T06:24:42.677 INFO:teuthology.orchestra.run.smithi089.stdout:Depends: linux-image-5.15.0-113-generic, linux-modules-extra-5.15.0-113-generic, linux-firmware, intel-microcode, amd64-microcode 2024-07-15T06:24:42.677 INFO:teuthology.orchestra.run.smithi089.stdout:Recommends: thermald 2024-07-15T06:24:42.677 INFO:teuthology.orchestra.run.smithi089.stdout:Description: Generic Linux kernel image 2024-07-15T06:24:42.677 INFO:teuthology.orchestra.run.smithi089.stdout: This package will always depend on the latest generic kernel image 2024-07-15T06:24:42.677 INFO:teuthology.orchestra.run.smithi089.stdout: available. 2024-07-15T06:24:42.677 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.15.0-56-generic Expected: 5.15.0-113-generic 2024-07-15T06:24:42.677 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-07-15T06:24:42.678 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-07-15T06:24:42.678 INFO:teuthology.task.kernel:Installing distro kernel on mon.a... 2024-07-15T06:24:42.678 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi089.front.sepia.ceph.com, path=None, version=distro) 2024-07-15T06:24:42.678 DEBUG:teuthology.orchestra.run.smithi089:> sudo apt-get clean 2024-07-15T06:24:42.856 DEBUG:teuthology.orchestra.run.smithi089:> sudo apt-get update 2024-07-15T06:24:43.119 INFO:teuthology.orchestra.run.smithi089.stdout:Hit:1 http://archive.ubuntu.com/ubuntu jammy InRelease 2024-07-15T06:24:43.181 INFO:teuthology.orchestra.run.smithi089.stdout:Hit:2 http://archive.ubuntu.com/ubuntu jammy-updates InRelease 2024-07-15T06:24:43.201 INFO:teuthology.orchestra.run.smithi089.stdout:Hit:3 http://security.ubuntu.com/ubuntu jammy-security InRelease 2024-07-15T06:24:43.203 INFO:teuthology.orchestra.run.smithi134.stdout:/etc/kernel/postinst.d/zz-update-grub: 2024-07-15T06:24:43.204 INFO:teuthology.orchestra.run.smithi134.stdout:Sourcing file `/etc/default/grub' 2024-07-15T06:24:43.221 INFO:teuthology.orchestra.run.smithi134.stdout:Sourcing file `/etc/default/grub.d/init-select.cfg' 2024-07-15T06:24:43.240 INFO:teuthology.orchestra.run.smithi134.stdout:Generating grub configuration file ... 2024-07-15T06:24:43.264 INFO:teuthology.orchestra.run.smithi089.stdout:Hit:4 http://archive.ubuntu.com/ubuntu jammy-backports InRelease 2024-07-15T06:24:43.607 INFO:teuthology.orchestra.run.smithi134.stdout:Found linux image: /boot/vmlinuz-5.15.0-113-generic 2024-07-15T06:24:43.628 INFO:teuthology.orchestra.run.smithi134.stdout:Found initrd image: /boot/initrd.img-5.15.0-113-generic 2024-07-15T06:24:44.022 INFO:teuthology.orchestra.run.smithi134.stdout:Found linux image: /boot/vmlinuz-5.15.0-56-generic 2024-07-15T06:24:44.028 INFO:teuthology.orchestra.run.smithi134.stdout:Found initrd image: /boot/initrd.img-5.15.0-56-generic 2024-07-15T06:24:44.100 INFO:teuthology.orchestra.run.smithi134.stdout:Found linux image: /boot/vmlinuz-5.4.0-135-generic 2024-07-15T06:24:44.105 INFO:teuthology.orchestra.run.smithi134.stdout:Found initrd image: /boot/initrd.img-5.4.0-135-generic 2024-07-15T06:24:44.259 INFO:teuthology.orchestra.run.smithi134.stdout:Warning: os-prober will not be executed to detect other bootable partitions. 2024-07-15T06:24:44.259 INFO:teuthology.orchestra.run.smithi134.stdout:Systems on them will not be added to the GRUB boot configuration. 2024-07-15T06:24:44.259 INFO:teuthology.orchestra.run.smithi134.stdout:Check GRUB_DISABLE_OS_PROBER documentation entry. 2024-07-15T06:24:44.299 INFO:teuthology.orchestra.run.smithi134.stdout:done 2024-07-15T06:24:44.563 INFO:teuthology.orchestra.run.smithi089.stdout:Reading package lists... 2024-07-15T06:24:44.581 DEBUG:teuthology.orchestra.run.smithi089:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic 2024-07-15T06:24:44.636 INFO:teuthology.orchestra.run.smithi089.stdout:Reading package lists... 2024-07-15T06:24:44.780 DEBUG:teuthology.orchestra.run.smithi134:> dpkg -s linux-image-generic 2024-07-15T06:24:44.797 INFO:teuthology.orchestra.run.smithi134.stdout:Package: linux-image-generic 2024-07-15T06:24:44.797 INFO:teuthology.orchestra.run.smithi134.stdout:Status: install ok installed 2024-07-15T06:24:44.797 INFO:teuthology.orchestra.run.smithi134.stdout:Priority: optional 2024-07-15T06:24:44.797 INFO:teuthology.orchestra.run.smithi134.stdout:Section: kernel 2024-07-15T06:24:44.797 INFO:teuthology.orchestra.run.smithi134.stdout:Installed-Size: 21 2024-07-15T06:24:44.797 INFO:teuthology.orchestra.run.smithi134.stdout:Maintainer: Ubuntu Kernel Team 2024-07-15T06:24:44.797 INFO:teuthology.orchestra.run.smithi134.stdout:Architecture: amd64 2024-07-15T06:24:44.798 INFO:teuthology.orchestra.run.smithi134.stdout:Source: linux-meta 2024-07-15T06:24:44.798 INFO:teuthology.orchestra.run.smithi134.stdout:Version: 5.15.0.113.113 2024-07-15T06:24:44.798 INFO:teuthology.orchestra.run.smithi134.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.3), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-113), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.3) 2024-07-15T06:24:44.798 INFO:teuthology.orchestra.run.smithi134.stdout:Depends: linux-image-5.15.0-113-generic, linux-modules-extra-5.15.0-113-generic, linux-firmware, intel-microcode, amd64-microcode 2024-07-15T06:24:44.798 INFO:teuthology.orchestra.run.smithi134.stdout:Recommends: thermald 2024-07-15T06:24:44.798 INFO:teuthology.orchestra.run.smithi134.stdout:Description: Generic Linux kernel image 2024-07-15T06:24:44.798 INFO:teuthology.orchestra.run.smithi134.stdout: This package will always depend on the latest generic kernel image 2024-07-15T06:24:44.798 INFO:teuthology.orchestra.run.smithi134.stdout: available. 2024-07-15T06:24:44.798 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.15.0-56-generic Expected: 5.15.0-113-generic 2024-07-15T06:24:44.798 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-07-15T06:24:44.799 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-07-15T06:24:44.799 INFO:teuthology.task.kernel:Installing distro kernel on mon.b... 2024-07-15T06:24:44.799 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi134.front.sepia.ceph.com, path=None, version=distro) 2024-07-15T06:24:44.799 DEBUG:teuthology.orchestra.run.smithi134:> sudo apt-get clean 2024-07-15T06:24:44.818 INFO:teuthology.orchestra.run.smithi089.stdout:Building dependency tree... 2024-07-15T06:24:44.822 INFO:teuthology.orchestra.run.smithi089.stdout:Reading state information... 2024-07-15T06:24:44.977 DEBUG:teuthology.orchestra.run.smithi134:> sudo apt-get update 2024-07-15T06:24:44.979 INFO:teuthology.orchestra.run.smithi089.stdout:linux-image-generic is already the newest version (5.15.0.113.113). 2024-07-15T06:24:44.979 INFO:teuthology.orchestra.run.smithi089.stdout:The following packages were automatically installed and are no longer required: 2024-07-15T06:24:44.979 INFO:teuthology.orchestra.run.smithi089.stdout: libboost-iostreams1.74.0 libboost-thread1.74.0 libgfapi0 libgfrpc0 libgfxdr0 2024-07-15T06:24:44.979 INFO:teuthology.orchestra.run.smithi089.stdout: libglusterfs0 libiscsi7 libpmemobj1 libpython2-dev libpython2-stdlib 2024-07-15T06:24:44.979 INFO:teuthology.orchestra.run.smithi089.stdout: libpython2.7 libpython2.7-dev libpython2.7-minimal libpython2.7-stdlib 2024-07-15T06:24:44.979 INFO:teuthology.orchestra.run.smithi089.stdout: linux-image-5.4.0-135-generic linux-modules-5.4.0-135-generic python2 2024-07-15T06:24:44.980 INFO:teuthology.orchestra.run.smithi089.stdout: python2-dev python2-minimal python2.7 python2.7-dev python2.7-minimal 2024-07-15T06:24:44.980 INFO:teuthology.orchestra.run.smithi089.stdout:Use 'sudo apt autoremove' to remove them. 2024-07-15T06:24:45.019 INFO:teuthology.orchestra.run.smithi089.stdout:0 upgraded, 0 newly installed, 0 to remove and 329 not upgraded. 2024-07-15T06:24:45.021 DEBUG:teuthology.orchestra.run.smithi089:> dpkg -s linux-image-generic 2024-07-15T06:24:45.077 INFO:teuthology.orchestra.run.smithi089.stdout:Package: linux-image-generic 2024-07-15T06:24:45.077 INFO:teuthology.orchestra.run.smithi089.stdout:Status: install ok installed 2024-07-15T06:24:45.077 INFO:teuthology.orchestra.run.smithi089.stdout:Priority: optional 2024-07-15T06:24:45.077 INFO:teuthology.orchestra.run.smithi089.stdout:Section: kernel 2024-07-15T06:24:45.077 INFO:teuthology.orchestra.run.smithi089.stdout:Installed-Size: 21 2024-07-15T06:24:45.077 INFO:teuthology.orchestra.run.smithi089.stdout:Maintainer: Ubuntu Kernel Team 2024-07-15T06:24:45.077 INFO:teuthology.orchestra.run.smithi089.stdout:Architecture: amd64 2024-07-15T06:24:45.078 INFO:teuthology.orchestra.run.smithi089.stdout:Source: linux-meta 2024-07-15T06:24:45.078 INFO:teuthology.orchestra.run.smithi089.stdout:Version: 5.15.0.113.113 2024-07-15T06:24:45.078 INFO:teuthology.orchestra.run.smithi089.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.3), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-113), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.3) 2024-07-15T06:24:45.078 INFO:teuthology.orchestra.run.smithi089.stdout:Depends: linux-image-5.15.0-113-generic, linux-modules-extra-5.15.0-113-generic, linux-firmware, intel-microcode, amd64-microcode 2024-07-15T06:24:45.078 INFO:teuthology.orchestra.run.smithi089.stdout:Recommends: thermald 2024-07-15T06:24:45.078 INFO:teuthology.orchestra.run.smithi089.stdout:Description: Generic Linux kernel image 2024-07-15T06:24:45.078 INFO:teuthology.orchestra.run.smithi089.stdout: This package will always depend on the latest generic kernel image 2024-07-15T06:24:45.078 INFO:teuthology.orchestra.run.smithi089.stdout: available. 2024-07-15T06:24:45.078 DEBUG:teuthology.orchestra.run.smithi089:> mktemp 2024-07-15T06:24:45.092 INFO:teuthology.orchestra.run.smithi134.stdout:Hit:1 http://archive.ubuntu.com/ubuntu jammy InRelease 2024-07-15T06:24:45.100 INFO:teuthology.orchestra.run.smithi134.stdout:Hit:2 http://archive.ubuntu.com/ubuntu jammy-updates InRelease 2024-07-15T06:24:45.117 INFO:teuthology.orchestra.run.smithi134.stdout:Hit:3 http://archive.ubuntu.com/ubuntu jammy-backports InRelease 2024-07-15T06:24:45.122 INFO:teuthology.orchestra.run.smithi089.stdout:/tmp/tmp.XM6gb15lqe 2024-07-15T06:24:45.122 DEBUG:teuthology.orchestra.run.smithi089:> sudo cp /boot/grub/grub.cfg /tmp/tmp.XM6gb15lqe 2024-07-15T06:24:45.178 DEBUG:teuthology.orchestra.run.smithi089:> sudo chmod 0666 /tmp/tmp.XM6gb15lqe 2024-07-15T06:24:45.211 INFO:teuthology.orchestra.run.smithi134.stdout:Hit:4 http://security.ubuntu.com/ubuntu jammy-security InRelease 2024-07-15T06:24:45.298 DEBUG:teuthology.orchestra.remote:smithi089:/tmp/tmp.XM6gb15lqe is 10KB 2024-07-15T06:24:45.310 DEBUG:teuthology.orchestra.run.smithi089:> rm -fr /tmp/tmp.XM6gb15lqe 2024-07-15T06:24:45.316 DEBUG:teuthology.orchestra.run.smithi089:> sudo rm -f -- /etc/grub.d/01_ceph_kernel 2024-07-15T06:24:45.370 DEBUG:teuthology.orchestra.run.smithi089:> set -ex 2024-07-15T06:24:45.370 DEBUG:teuthology.orchestra.run.smithi089:> sudo dd of=/etc/grub.d/01_ceph_kernel 2024-07-15T06:24:45.370 DEBUG:teuthology.orchestra.run.smithi089:> sudo chmod 755 /etc/grub.d/01_ceph_kernel 2024-07-15T06:24:45.452 INFO:teuthology.task.kernel:Distro Kernel Version: 5.15.0-113-generic 2024-07-15T06:24:45.452 DEBUG:teuthology.orchestra.run.smithi089:> sudo update-grub 2024-07-15T06:24:46.457 INFO:teuthology.orchestra.run.smithi134.stdout:Reading package lists... 2024-07-15T06:24:46.476 DEBUG:teuthology.orchestra.run.smithi134:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic 2024-07-15T06:24:46.538 INFO:teuthology.orchestra.run.smithi134.stdout:Reading package lists... 2024-07-15T06:24:46.707 INFO:teuthology.orchestra.run.smithi089.stderr:Sourcing file `/etc/default/grub' 2024-07-15T06:24:46.708 INFO:teuthology.orchestra.run.smithi089.stderr:Sourcing file `/etc/default/grub.d/init-select.cfg' 2024-07-15T06:24:46.711 INFO:teuthology.orchestra.run.smithi089.stderr:Generating grub configuration file ... 2024-07-15T06:24:46.741 INFO:teuthology.orchestra.run.smithi134.stdout:Building dependency tree... 2024-07-15T06:24:46.742 INFO:teuthology.orchestra.run.smithi134.stdout:Reading state information... 2024-07-15T06:24:46.914 INFO:teuthology.orchestra.run.smithi134.stdout:linux-image-generic is already the newest version (5.15.0.113.113). 2024-07-15T06:24:46.915 INFO:teuthology.orchestra.run.smithi134.stdout:The following packages were automatically installed and are no longer required: 2024-07-15T06:24:46.915 INFO:teuthology.orchestra.run.smithi134.stdout: libboost-iostreams1.74.0 libboost-thread1.74.0 libgfapi0 libgfrpc0 libgfxdr0 2024-07-15T06:24:46.915 INFO:teuthology.orchestra.run.smithi134.stdout: libglusterfs0 libiscsi7 libpmemobj1 libpython2-dev libpython2-stdlib 2024-07-15T06:24:46.915 INFO:teuthology.orchestra.run.smithi134.stdout: libpython2.7 libpython2.7-dev libpython2.7-minimal libpython2.7-stdlib 2024-07-15T06:24:46.915 INFO:teuthology.orchestra.run.smithi134.stdout: linux-image-5.4.0-135-generic linux-modules-5.4.0-135-generic python2 2024-07-15T06:24:46.915 INFO:teuthology.orchestra.run.smithi134.stdout: python2-dev python2-minimal python2.7 python2.7-dev python2.7-minimal 2024-07-15T06:24:46.915 INFO:teuthology.orchestra.run.smithi134.stdout:Use 'sudo apt autoremove' to remove them. 2024-07-15T06:24:46.945 INFO:teuthology.orchestra.run.smithi089.stderr:Found linux image: /boot/vmlinuz-5.15.0-113-generic 2024-07-15T06:24:46.950 INFO:teuthology.orchestra.run.smithi089.stderr:Found initrd image: /boot/initrd.img-5.15.0-113-generic 2024-07-15T06:24:46.954 INFO:teuthology.orchestra.run.smithi134.stdout:0 upgraded, 0 newly installed, 0 to remove and 329 not upgraded. 2024-07-15T06:24:46.957 DEBUG:teuthology.orchestra.run.smithi134:> dpkg -s linux-image-generic 2024-07-15T06:24:46.969 INFO:teuthology.orchestra.run.smithi134.stdout:Package: linux-image-generic 2024-07-15T06:24:46.969 INFO:teuthology.orchestra.run.smithi134.stdout:Status: install ok installed 2024-07-15T06:24:46.969 INFO:teuthology.orchestra.run.smithi134.stdout:Priority: optional 2024-07-15T06:24:46.969 INFO:teuthology.orchestra.run.smithi134.stdout:Section: kernel 2024-07-15T06:24:46.969 INFO:teuthology.orchestra.run.smithi134.stdout:Installed-Size: 21 2024-07-15T06:24:46.970 INFO:teuthology.orchestra.run.smithi134.stdout:Maintainer: Ubuntu Kernel Team 2024-07-15T06:24:46.970 INFO:teuthology.orchestra.run.smithi134.stdout:Architecture: amd64 2024-07-15T06:24:46.970 INFO:teuthology.orchestra.run.smithi134.stdout:Source: linux-meta 2024-07-15T06:24:46.970 INFO:teuthology.orchestra.run.smithi134.stdout:Version: 5.15.0.113.113 2024-07-15T06:24:46.970 INFO:teuthology.orchestra.run.smithi134.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.3), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-113), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.3) 2024-07-15T06:24:46.970 INFO:teuthology.orchestra.run.smithi134.stdout:Depends: linux-image-5.15.0-113-generic, linux-modules-extra-5.15.0-113-generic, linux-firmware, intel-microcode, amd64-microcode 2024-07-15T06:24:46.970 INFO:teuthology.orchestra.run.smithi134.stdout:Recommends: thermald 2024-07-15T06:24:46.970 INFO:teuthology.orchestra.run.smithi134.stdout:Description: Generic Linux kernel image 2024-07-15T06:24:46.970 INFO:teuthology.orchestra.run.smithi134.stdout: This package will always depend on the latest generic kernel image 2024-07-15T06:24:46.970 INFO:teuthology.orchestra.run.smithi134.stdout: available. 2024-07-15T06:24:46.971 DEBUG:teuthology.orchestra.run.smithi134:> mktemp 2024-07-15T06:24:47.018 INFO:teuthology.orchestra.run.smithi134.stdout:/tmp/tmp.uR08PwhAq4 2024-07-15T06:24:47.018 DEBUG:teuthology.orchestra.run.smithi134:> sudo cp /boot/grub/grub.cfg /tmp/tmp.uR08PwhAq4 2024-07-15T06:24:47.085 DEBUG:teuthology.orchestra.run.smithi134:> sudo chmod 0666 /tmp/tmp.uR08PwhAq4 2024-07-15T06:24:47.177 INFO:teuthology.orchestra.run.smithi089.stderr:Found linux image: /boot/vmlinuz-5.15.0-56-generic 2024-07-15T06:24:47.183 INFO:teuthology.orchestra.run.smithi089.stderr:Found initrd image: /boot/initrd.img-5.15.0-56-generic 2024-07-15T06:24:47.218 DEBUG:teuthology.orchestra.remote:smithi134:/tmp/tmp.uR08PwhAq4 is 10KB 2024-07-15T06:24:47.223 INFO:teuthology.orchestra.run.smithi089.stderr:Found linux image: /boot/vmlinuz-5.4.0-135-generic 2024-07-15T06:24:47.228 INFO:teuthology.orchestra.run.smithi089.stderr:Found initrd image: /boot/initrd.img-5.4.0-135-generic 2024-07-15T06:24:47.231 DEBUG:teuthology.orchestra.run.smithi134:> rm -fr /tmp/tmp.uR08PwhAq4 2024-07-15T06:24:47.236 DEBUG:teuthology.orchestra.run.smithi134:> sudo rm -f -- /etc/grub.d/01_ceph_kernel 2024-07-15T06:24:47.288 DEBUG:teuthology.orchestra.run.smithi134:> set -ex 2024-07-15T06:24:47.288 DEBUG:teuthology.orchestra.run.smithi134:> sudo dd of=/etc/grub.d/01_ceph_kernel 2024-07-15T06:24:47.289 DEBUG:teuthology.orchestra.run.smithi134:> sudo chmod 755 /etc/grub.d/01_ceph_kernel 2024-07-15T06:24:47.344 INFO:teuthology.orchestra.run.smithi089.stderr:Warning: os-prober will not be executed to detect other bootable partitions. 2024-07-15T06:24:47.344 INFO:teuthology.orchestra.run.smithi089.stderr:Systems on them will not be added to the GRUB boot configuration. 2024-07-15T06:24:47.344 INFO:teuthology.orchestra.run.smithi089.stderr:Check GRUB_DISABLE_OS_PROBER documentation entry. 2024-07-15T06:24:47.360 INFO:teuthology.task.kernel:Distro Kernel Version: 5.15.0-113-generic 2024-07-15T06:24:47.360 DEBUG:teuthology.orchestra.run.smithi134:> sudo update-grub 2024-07-15T06:24:47.375 INFO:teuthology.orchestra.run.smithi089.stderr:done 2024-07-15T06:24:47.376 DEBUG:teuthology.orchestra.run.smithi089:> sudo shutdown -r now 2024-07-15T06:24:48.635 INFO:teuthology.orchestra.run.smithi134.stderr:Sourcing file `/etc/default/grub' 2024-07-15T06:24:48.636 INFO:teuthology.orchestra.run.smithi134.stderr:Sourcing file `/etc/default/grub.d/init-select.cfg' 2024-07-15T06:24:48.639 INFO:teuthology.orchestra.run.smithi134.stderr:Generating grub configuration file ... 2024-07-15T06:24:48.911 INFO:teuthology.orchestra.run.smithi134.stderr:Found linux image: /boot/vmlinuz-5.15.0-113-generic 2024-07-15T06:24:48.918 INFO:teuthology.orchestra.run.smithi134.stderr:Found initrd image: /boot/initrd.img-5.15.0-113-generic 2024-07-15T06:24:49.196 INFO:teuthology.orchestra.run.smithi134.stderr:Found linux image: /boot/vmlinuz-5.15.0-56-generic 2024-07-15T06:24:49.201 INFO:teuthology.orchestra.run.smithi134.stderr:Found initrd image: /boot/initrd.img-5.15.0-56-generic 2024-07-15T06:24:49.241 INFO:teuthology.orchestra.run.smithi134.stderr:Found linux image: /boot/vmlinuz-5.4.0-135-generic 2024-07-15T06:24:49.247 INFO:teuthology.orchestra.run.smithi134.stderr:Found initrd image: /boot/initrd.img-5.4.0-135-generic 2024-07-15T06:24:49.368 INFO:teuthology.orchestra.run.smithi134.stderr:Warning: os-prober will not be executed to detect other bootable partitions. 2024-07-15T06:24:49.368 INFO:teuthology.orchestra.run.smithi134.stderr:Systems on them will not be added to the GRUB boot configuration. 2024-07-15T06:24:49.368 INFO:teuthology.orchestra.run.smithi134.stderr:Check GRUB_DISABLE_OS_PROBER documentation entry. 2024-07-15T06:24:49.389 INFO:teuthology.orchestra.run.smithi134.stderr:done 2024-07-15T06:24:49.391 DEBUG:teuthology.orchestra.run.smithi134:> sudo shutdown -r now 2024-07-15T06:25:17.406 INFO:teuthology.task.kernel:Checking client mon.a for new kernel version... 2024-07-15T06:25:17.407 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi089.front.sepia.ceph.com' 2024-07-15T06:25:17.408 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi089.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-07-15T06:25:19.394 INFO:teuthology.task.kernel:Checking client mon.b for new kernel version... 2024-07-15T06:25:19.394 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi134.front.sepia.ceph.com' 2024-07-15T06:25:19.395 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi134.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-07-15T06:25:35.775 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.89 2024-07-15T06:25:37.826 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.134 2024-07-15T06:25:44.783 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi089.front.sepia.ceph.com' 2024-07-15T06:25:44.784 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi089.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-07-15T06:25:46.829 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi134.front.sepia.ceph.com' 2024-07-15T06:25:46.830 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi134.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-07-15T06:25:47.838 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.89 2024-07-15T06:25:49.890 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.134 2024-07-15T06:25:59.849 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi089.front.sepia.ceph.com' 2024-07-15T06:25:59.849 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi089.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-07-15T06:26:01.893 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi134.front.sepia.ceph.com' 2024-07-15T06:26:01.894 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi134.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-07-15T06:26:18.271 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.89 2024-07-15T06:26:20.318 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.134 2024-07-15T06:26:33.284 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi089.front.sepia.ceph.com' 2024-07-15T06:26:33.284 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi089.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-07-15T06:26:33.289 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.89 2024-07-15T06:26:35.319 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi134.front.sepia.ceph.com' 2024-07-15T06:26:35.320 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi134.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-07-15T06:26:36.072 DEBUG:teuthology.orchestra.run.smithi134:> true 2024-07-15T06:26:36.936 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi134.front.sepia.ceph.com' 2024-07-15T06:26:36.936 INFO:teuthology.task.kernel:Checking kernel version of mon.b, want "5.15.0-113-generic"... 2024-07-15T06:26:36.936 DEBUG:teuthology.orchestra.run.smithi134:> uname -r 2024-07-15T06:26:36.984 INFO:teuthology.orchestra.run.smithi134.stdout:5.15.0-113-generic 2024-07-15T06:26:37.054 DEBUG:teuthology.task.kernel:current kernel version is 5.15.0-113-generic vs 5.15.0-113-generic 2024-07-15T06:26:37.054 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-07-15T06:26:37.055 DEBUG:teuthology.task.kernel:Distro of this test job: ubuntu 2024-07-15T06:26:38.055 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2024-07-15T06:26:38.055 DEBUG:teuthology.orchestra.run.smithi134:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-07-15T06:26:38.140 INFO:teuthology.orchestra.run.smithi134.stdout:ttyS1 2024-07-15T06:26:38.165 DEBUG:teuthology.parallel:result is None 2024-07-15T06:26:51.289 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi089.front.sepia.ceph.com' 2024-07-15T06:26:51.290 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi089.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-07-15T06:26:51.558 DEBUG:teuthology.orchestra.run.smithi089:> true 2024-07-15T06:26:52.169 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi089.front.sepia.ceph.com' 2024-07-15T06:26:52.169 INFO:teuthology.task.kernel:Checking kernel version of mon.a, want "5.15.0-113-generic"... 2024-07-15T06:26:52.169 DEBUG:teuthology.orchestra.run.smithi089:> uname -r 2024-07-15T06:26:52.217 INFO:teuthology.orchestra.run.smithi089.stdout:5.15.0-113-generic 2024-07-15T06:26:52.218 DEBUG:teuthology.task.kernel:current kernel version is 5.15.0-113-generic vs 5.15.0-113-generic 2024-07-15T06:26:52.218 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-07-15T06:26:52.218 DEBUG:teuthology.task.kernel:Distro of this test job: ubuntu 2024-07-15T06:26:53.218 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2024-07-15T06:26:53.219 DEBUG:teuthology.orchestra.run.smithi089:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-07-15T06:26:53.564 INFO:teuthology.orchestra.run.smithi089.stdout:ttyS1 2024-07-15T06:26:53.587 DEBUG:teuthology.parallel:result is None 2024-07-15T06:26:53.587 INFO:teuthology.run_tasks:Running task internal.base... 2024-07-15T06:26:53.598 INFO:teuthology.task.internal:Creating test directory... 2024-07-15T06:26:53.598 DEBUG:teuthology.orchestra.run.smithi089:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-07-15T06:26:53.601 DEBUG:teuthology.orchestra.run.smithi134:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-07-15T06:26:53.608 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-07-15T06:26:53.620 INFO:teuthology.run_tasks:Running task internal.archive... 2024-07-15T06:26:53.634 INFO:teuthology.task.internal:Creating archive directory... 2024-07-15T06:26:53.634 DEBUG:teuthology.orchestra.run.smithi089:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-07-15T06:26:53.650 DEBUG:teuthology.orchestra.run.smithi134:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-07-15T06:26:53.680 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-07-15T06:26:53.690 INFO:teuthology.task.internal:Enabling coredump saving... 2024-07-15T06:26:53.691 DEBUG:teuthology.orchestra.run.smithi089:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-07-15T06:26:53.718 DEBUG:teuthology.orchestra.run.smithi134:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-07-15T06:26:53.735 INFO:teuthology.orchestra.run.smithi089.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-07-15T06:26:53.743 INFO:teuthology.orchestra.run.smithi134.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-07-15T06:26:53.743 INFO:teuthology.orchestra.run.smithi089.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-07-15T06:26:53.753 INFO:teuthology.orchestra.run.smithi134.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-07-15T06:26:53.754 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-07-15T06:26:53.765 INFO:teuthology.task.internal:Configuring sudo... 2024-07-15T06:26:53.766 DEBUG:teuthology.orchestra.run.smithi089:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-07-15T06:26:53.791 DEBUG:teuthology.orchestra.run.smithi134:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-07-15T06:26:53.808 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-07-15T06:26:53.817 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-07-15T06:26:53.817 DEBUG:teuthology.orchestra.run.smithi089:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-07-15T06:26:53.846 DEBUG:teuthology.orchestra.run.smithi134:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-07-15T06:26:53.855 DEBUG:teuthology.orchestra.run.smithi089:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-07-15T06:26:53.898 DEBUG:teuthology.orchestra.run.smithi089:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-07-15T06:26:53.946 DEBUG:teuthology.orchestra.run.smithi089:> set -ex 2024-07-15T06:26:53.946 DEBUG:teuthology.orchestra.run.smithi089:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-07-15T06:26:54.022 DEBUG:teuthology.orchestra.run.smithi134:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-07-15T06:26:54.027 DEBUG:teuthology.orchestra.run.smithi134:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-07-15T06:26:54.076 DEBUG:teuthology.orchestra.run.smithi134:> set -ex 2024-07-15T06:26:54.076 DEBUG:teuthology.orchestra.run.smithi134:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-07-15T06:26:54.135 DEBUG:teuthology.orchestra.run.smithi089:> sudo service rsyslog restart 2024-07-15T06:26:54.137 DEBUG:teuthology.orchestra.run.smithi134:> sudo service rsyslog restart 2024-07-15T06:26:54.207 INFO:teuthology.run_tasks:Running task internal.timer... 2024-07-15T06:26:54.217 INFO:teuthology.task.internal:Starting timer... 2024-07-15T06:26:54.217 INFO:teuthology.run_tasks:Running task pcp... 2024-07-15T06:26:54.234 INFO:teuthology.run_tasks:Running task selinux... 2024-07-15T06:26:54.246 DEBUG:teuthology.task.selinux:Excluding smithi089: OS 'ubuntu' does not support SELinux 2024-07-15T06:26:54.246 DEBUG:teuthology.task.selinux:Excluding smithi134: OS 'ubuntu' does not support SELinux 2024-07-15T06:26:54.246 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-07-15T06:26:54.246 DEBUG:teuthology.task.selinux:Existing SELinux modes: {} 2024-07-15T06:26:54.246 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-07-15T06:26:54.246 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-07-15T06:26:54.257 DEBUG:teuthology.repo_utils:Setting repo remote to https://git.ceph.com/ceph-cm-ansible.git 2024-07-15T06:26:54.260 INFO:teuthology.repo_utils:Fetching git.ceph.com_ceph-cm-ansible_main from origin 2024-07-15T06:26:54.369 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-07-15T06:26:54.380 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-07-15T06:26:54.381 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi089.front.sepia.ceph.com,smithi134.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-07-15T06:33:54.813 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi089.front.sepia.ceph.com'), Remote(name='ubuntu@smithi134.front.sepia.ceph.com')] 2024-07-15T06:33:54.815 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi089.front.sepia.ceph.com' 2024-07-15T06:33:54.816 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi089.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-07-15T06:33:54.902 DEBUG:teuthology.orchestra.run.smithi089:> true 2024-07-15T06:33:54.978 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi089.front.sepia.ceph.com' 2024-07-15T06:33:54.979 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi134.front.sepia.ceph.com' 2024-07-15T06:33:54.979 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi134.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-07-15T06:33:55.064 DEBUG:teuthology.orchestra.run.smithi134:> true 2024-07-15T06:33:55.138 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi134.front.sepia.ceph.com' 2024-07-15T06:33:55.138 INFO:teuthology.run_tasks:Running task clock... 2024-07-15T06:33:55.152 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-07-15T06:33:55.153 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-07-15T06:33:55.153 DEBUG:teuthology.orchestra.run.smithi089:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-07-15T06:33:55.156 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-07-15T06:33:55.156 DEBUG:teuthology.orchestra.run.smithi134:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-07-15T06:33:55.183 INFO:teuthology.orchestra.run.smithi089.stdout:15 Jul 06:33:55 ntpd[17654]: ntpd 4.2.8p15@1.3728-o Wed Feb 16 17:13:02 UTC 2022 (1): Starting 2024-07-15T06:33:55.183 INFO:teuthology.orchestra.run.smithi089.stdout:15 Jul 06:33:55 ntpd[17654]: Command line: ntpd -gq 2024-07-15T06:33:55.184 INFO:teuthology.orchestra.run.smithi089.stdout:15 Jul 06:33:55 ntpd[17654]: ---------------------------------------------------- 2024-07-15T06:33:55.184 INFO:teuthology.orchestra.run.smithi089.stdout:15 Jul 06:33:55 ntpd[17654]: ntp-4 is maintained by Network Time Foundation, 2024-07-15T06:33:55.184 INFO:teuthology.orchestra.run.smithi089.stdout:15 Jul 06:33:55 ntpd[17654]: Inc. (NTF), a non-profit 501(c)(3) public-benefit 2024-07-15T06:33:55.184 INFO:teuthology.orchestra.run.smithi089.stdout:15 Jul 06:33:55 ntpd[17654]: corporation. Support and training for ntp-4 are 2024-07-15T06:33:55.184 INFO:teuthology.orchestra.run.smithi089.stdout:15 Jul 06:33:55 ntpd[17654]: available at https://www.nwtime.org/support 2024-07-15T06:33:55.184 INFO:teuthology.orchestra.run.smithi089.stdout:15 Jul 06:33:55 ntpd[17654]: ---------------------------------------------------- 2024-07-15T06:33:55.185 INFO:teuthology.orchestra.run.smithi089.stdout:15 Jul 06:33:55 ntpd[17654]: proto: precision = 0.047 usec (-24) 2024-07-15T06:33:55.185 INFO:teuthology.orchestra.run.smithi089.stdout:15 Jul 06:33:55 ntpd[17654]: basedate set to 2022-02-04 2024-07-15T06:33:55.185 INFO:teuthology.orchestra.run.smithi089.stdout:15 Jul 06:33:55 ntpd[17654]: gps base set to 2022-02-06 (week 2196) 2024-07-15T06:33:55.185 INFO:teuthology.orchestra.run.smithi089.stdout:15 Jul 06:33:55 ntpd[17654]: restrict 0.0.0.0: KOD does nothing without LIMITED. 2024-07-15T06:33:55.185 INFO:teuthology.orchestra.run.smithi089.stderr:restrict 0.0.0.0: KOD does nothing without LIMITED. 2024-07-15T06:33:55.185 INFO:teuthology.orchestra.run.smithi089.stderr:restrict ::: KOD does nothing without LIMITED. 2024-07-15T06:33:55.186 INFO:teuthology.orchestra.run.smithi089.stdout:15 Jul 06:33:55 ntpd[17654]: restrict ::: KOD does nothing without LIMITED. 2024-07-15T06:33:55.186 INFO:teuthology.orchestra.run.smithi089.stdout:15 Jul 06:33:55 ntpd[17654]: Listen and drop on 0 v6wildcard [::]:123 2024-07-15T06:33:55.186 INFO:teuthology.orchestra.run.smithi089.stdout:15 Jul 06:33:55 ntpd[17654]: Listen and drop on 1 v4wildcard 0.0.0.0:123 2024-07-15T06:33:55.187 INFO:teuthology.orchestra.run.smithi089.stdout:15 Jul 06:33:55 ntpd[17654]: Listen normally on 2 lo 127.0.0.1:123 2024-07-15T06:33:55.187 INFO:teuthology.orchestra.run.smithi089.stdout:15 Jul 06:33:55 ntpd[17654]: Listen normally on 3 enp3s0f1 172.21.15.89:123 2024-07-15T06:33:55.187 INFO:teuthology.orchestra.run.smithi089.stdout:15 Jul 06:33:55 ntpd[17654]: Listen normally on 4 lo [::1]:123 2024-07-15T06:33:55.187 INFO:teuthology.orchestra.run.smithi089.stdout:15 Jul 06:33:55 ntpd[17654]: Listen normally on 5 enp3s0f1 [fe80::ec4:7aff:fe8f:d501%5]:123 2024-07-15T06:33:55.187 INFO:teuthology.orchestra.run.smithi089.stdout:15 Jul 06:33:55 ntpd[17654]: Listening on routing socket on fd #22 for interface updates 2024-07-15T06:33:55.208 INFO:teuthology.orchestra.run.smithi134.stdout:15 Jul 06:33:55 ntpd[17652]: ntpd 4.2.8p15@1.3728-o Wed Feb 16 17:13:02 UTC 2022 (1): Starting 2024-07-15T06:33:55.208 INFO:teuthology.orchestra.run.smithi134.stdout:15 Jul 06:33:55 ntpd[17652]: Command line: ntpd -gq 2024-07-15T06:33:55.208 INFO:teuthology.orchestra.run.smithi134.stdout:15 Jul 06:33:55 ntpd[17652]: ---------------------------------------------------- 2024-07-15T06:33:55.208 INFO:teuthology.orchestra.run.smithi134.stdout:15 Jul 06:33:55 ntpd[17652]: ntp-4 is maintained by Network Time Foundation, 2024-07-15T06:33:55.208 INFO:teuthology.orchestra.run.smithi134.stdout:15 Jul 06:33:55 ntpd[17652]: Inc. (NTF), a non-profit 501(c)(3) public-benefit 2024-07-15T06:33:55.208 INFO:teuthology.orchestra.run.smithi134.stdout:15 Jul 06:33:55 ntpd[17652]: corporation. Support and training for ntp-4 are 2024-07-15T06:33:55.208 INFO:teuthology.orchestra.run.smithi134.stdout:15 Jul 06:33:55 ntpd[17652]: available at https://www.nwtime.org/support 2024-07-15T06:33:55.209 INFO:teuthology.orchestra.run.smithi134.stdout:15 Jul 06:33:55 ntpd[17652]: ---------------------------------------------------- 2024-07-15T06:33:55.209 INFO:teuthology.orchestra.run.smithi134.stdout:15 Jul 06:33:55 ntpd[17652]: proto: precision = 0.044 usec (-24) 2024-07-15T06:33:55.209 INFO:teuthology.orchestra.run.smithi134.stdout:15 Jul 06:33:55 ntpd[17652]: basedate set to 2022-02-04 2024-07-15T06:33:55.209 INFO:teuthology.orchestra.run.smithi134.stdout:15 Jul 06:33:55 ntpd[17652]: gps base set to 2022-02-06 (week 2196) 2024-07-15T06:33:55.209 INFO:teuthology.orchestra.run.smithi134.stderr:restrict 0.0.0.0: KOD does nothing without LIMITED. 2024-07-15T06:33:55.209 INFO:teuthology.orchestra.run.smithi134.stdout:15 Jul 06:33:55 ntpd[17652]: restrict 0.0.0.0: KOD does nothing without LIMITED. 2024-07-15T06:33:55.209 INFO:teuthology.orchestra.run.smithi134.stdout:15 Jul 06:33:55 ntpd[17652]: restrict ::: KOD does nothing without LIMITED. 2024-07-15T06:33:55.210 INFO:teuthology.orchestra.run.smithi134.stderr:restrict ::: KOD does nothing without LIMITED. 2024-07-15T06:33:55.210 INFO:teuthology.orchestra.run.smithi134.stdout:15 Jul 06:33:55 ntpd[17652]: Listen and drop on 0 v6wildcard [::]:123 2024-07-15T06:33:55.211 INFO:teuthology.orchestra.run.smithi134.stdout:15 Jul 06:33:55 ntpd[17652]: Listen and drop on 1 v4wildcard 0.0.0.0:123 2024-07-15T06:33:55.211 INFO:teuthology.orchestra.run.smithi134.stdout:15 Jul 06:33:55 ntpd[17652]: Listen normally on 2 lo 127.0.0.1:123 2024-07-15T06:33:55.211 INFO:teuthology.orchestra.run.smithi134.stdout:15 Jul 06:33:55 ntpd[17652]: Listen normally on 3 enp3s0f1 172.21.15.134:123 2024-07-15T06:33:55.211 INFO:teuthology.orchestra.run.smithi134.stdout:15 Jul 06:33:55 ntpd[17652]: Listen normally on 4 lo [::1]:123 2024-07-15T06:33:55.211 INFO:teuthology.orchestra.run.smithi134.stdout:15 Jul 06:33:55 ntpd[17652]: Listen normally on 5 enp3s0f1 [fe80::ec4:7aff:fe88:6de9%5]:123 2024-07-15T06:33:55.211 INFO:teuthology.orchestra.run.smithi134.stdout:15 Jul 06:33:55 ntpd[17652]: Listening on routing socket on fd #22 for interface updates 2024-07-15T06:33:56.185 INFO:teuthology.orchestra.run.smithi089.stderr:15 Jul 06:33:56 ntpd[17654]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-07-15T06:33:56.185 INFO:teuthology.orchestra.run.smithi089.stderr:15 Jul 06:33:56 ntpd[17654]: can't open /var/log/ntpstats/rawstats.20240715: Permission denied 2024-07-15T06:33:56.185 INFO:teuthology.orchestra.run.smithi089.stderr:15 Jul 06:33:56 ntpd[17654]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-07-15T06:33:56.185 INFO:teuthology.orchestra.run.smithi089.stderr:15 Jul 06:33:56 ntpd[17654]: can't open /var/log/ntpstats/rawstats.20240715: Permission denied 2024-07-15T06:33:56.185 INFO:teuthology.orchestra.run.smithi089.stderr:15 Jul 06:33:56 ntpd[17654]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-07-15T06:33:56.185 INFO:teuthology.orchestra.run.smithi089.stderr:15 Jul 06:33:56 ntpd[17654]: can't open /var/log/ntpstats/rawstats.20240715: Permission denied 2024-07-15T06:33:56.185 INFO:teuthology.orchestra.run.smithi089.stderr:15 Jul 06:33:56 ntpd[17654]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-07-15T06:33:56.185 INFO:teuthology.orchestra.run.smithi089.stderr:15 Jul 06:33:56 ntpd[17654]: can't open /var/log/ntpstats/peerstats.20240715: Permission denied 2024-07-15T06:33:56.185 INFO:teuthology.orchestra.run.smithi089.stderr:15 Jul 06:33:56 ntpd[17654]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-07-15T06:33:56.185 INFO:teuthology.orchestra.run.smithi089.stderr:15 Jul 06:33:56 ntpd[17654]: can't open /var/log/ntpstats/rawstats.20240715: Permission denied 2024-07-15T06:33:56.186 INFO:teuthology.orchestra.run.smithi089.stderr:15 Jul 06:33:56 ntpd[17654]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-07-15T06:33:56.186 INFO:teuthology.orchestra.run.smithi089.stderr:15 Jul 06:33:56 ntpd[17654]: can't open /var/log/ntpstats/peerstats.20240715: Permission denied 2024-07-15T06:33:56.208 INFO:teuthology.orchestra.run.smithi134.stderr:15 Jul 06:33:56 ntpd[17652]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-07-15T06:33:56.208 INFO:teuthology.orchestra.run.smithi134.stderr:15 Jul 06:33:56 ntpd[17652]: can't open /var/log/ntpstats/rawstats.20240715: Permission denied 2024-07-15T06:33:56.208 INFO:teuthology.orchestra.run.smithi134.stderr:15 Jul 06:33:56 ntpd[17652]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-07-15T06:33:56.208 INFO:teuthology.orchestra.run.smithi134.stderr:15 Jul 06:33:56 ntpd[17652]: can't open /var/log/ntpstats/rawstats.20240715: Permission denied 2024-07-15T06:33:56.208 INFO:teuthology.orchestra.run.smithi134.stderr:15 Jul 06:33:56 ntpd[17652]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-07-15T06:33:56.208 INFO:teuthology.orchestra.run.smithi134.stderr:15 Jul 06:33:56 ntpd[17652]: can't open /var/log/ntpstats/peerstats.20240715: Permission denied 2024-07-15T06:33:58.185 INFO:teuthology.orchestra.run.smithi089.stderr:15 Jul 06:33:58 ntpd[17654]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-07-15T06:33:58.185 INFO:teuthology.orchestra.run.smithi089.stderr:15 Jul 06:33:58 ntpd[17654]: can't open /var/log/ntpstats/rawstats.20240715: Permission denied 2024-07-15T06:33:58.185 INFO:teuthology.orchestra.run.smithi089.stderr:15 Jul 06:33:58 ntpd[17654]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-07-15T06:33:58.185 INFO:teuthology.orchestra.run.smithi089.stderr:15 Jul 06:33:58 ntpd[17654]: can't open /var/log/ntpstats/rawstats.20240715: Permission denied 2024-07-15T06:33:58.185 INFO:teuthology.orchestra.run.smithi089.stderr:15 Jul 06:33:58 ntpd[17654]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-07-15T06:33:58.186 INFO:teuthology.orchestra.run.smithi089.stderr:15 Jul 06:33:58 ntpd[17654]: can't open /var/log/ntpstats/rawstats.20240715: Permission denied 2024-07-15T06:33:58.186 INFO:teuthology.orchestra.run.smithi089.stderr:15 Jul 06:33:58 ntpd[17654]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-07-15T06:33:58.186 INFO:teuthology.orchestra.run.smithi089.stderr:15 Jul 06:33:58 ntpd[17654]: can't open /var/log/ntpstats/rawstats.20240715: Permission denied 2024-07-15T06:33:58.186 INFO:teuthology.orchestra.run.smithi089.stderr:15 Jul 06:33:58 ntpd[17654]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-07-15T06:33:58.186 INFO:teuthology.orchestra.run.smithi089.stderr:15 Jul 06:33:58 ntpd[17654]: can't open /var/log/ntpstats/peerstats.20240715: Permission denied 2024-07-15T06:33:58.186 INFO:teuthology.orchestra.run.smithi089.stderr:15 Jul 06:33:58 ntpd[17654]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-07-15T06:33:58.186 INFO:teuthology.orchestra.run.smithi089.stderr:15 Jul 06:33:58 ntpd[17654]: can't open /var/log/ntpstats/rawstats.20240715: Permission denied 2024-07-15T06:33:58.186 INFO:teuthology.orchestra.run.smithi089.stderr:15 Jul 06:33:58 ntpd[17654]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-07-15T06:33:58.186 INFO:teuthology.orchestra.run.smithi089.stderr:15 Jul 06:33:58 ntpd[17654]: can't open /var/log/ntpstats/peerstats.20240715: Permission denied 2024-07-15T06:33:58.209 INFO:teuthology.orchestra.run.smithi134.stderr:15 Jul 06:33:58 ntpd[17652]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-07-15T06:33:58.209 INFO:teuthology.orchestra.run.smithi134.stderr:15 Jul 06:33:58 ntpd[17652]: can't open /var/log/ntpstats/rawstats.20240715: Permission denied 2024-07-15T06:33:58.209 INFO:teuthology.orchestra.run.smithi134.stderr:15 Jul 06:33:58 ntpd[17652]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-07-15T06:33:58.209 INFO:teuthology.orchestra.run.smithi134.stderr:15 Jul 06:33:58 ntpd[17652]: can't open /var/log/ntpstats/rawstats.20240715: Permission denied 2024-07-15T06:33:58.209 INFO:teuthology.orchestra.run.smithi134.stderr:15 Jul 06:33:58 ntpd[17652]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-07-15T06:33:58.209 INFO:teuthology.orchestra.run.smithi134.stderr:15 Jul 06:33:58 ntpd[17652]: can't open /var/log/ntpstats/rawstats.20240715: Permission denied 2024-07-15T06:33:58.209 INFO:teuthology.orchestra.run.smithi134.stderr:15 Jul 06:33:58 ntpd[17652]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-07-15T06:33:58.209 INFO:teuthology.orchestra.run.smithi134.stderr:15 Jul 06:33:58 ntpd[17652]: can't open /var/log/ntpstats/rawstats.20240715: Permission denied 2024-07-15T06:33:58.209 INFO:teuthology.orchestra.run.smithi134.stderr:15 Jul 06:33:58 ntpd[17652]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-07-15T06:33:58.209 INFO:teuthology.orchestra.run.smithi134.stderr:15 Jul 06:33:58 ntpd[17652]: can't open /var/log/ntpstats/peerstats.20240715: Permission denied 2024-07-15T06:33:58.210 INFO:teuthology.orchestra.run.smithi134.stderr:15 Jul 06:33:58 ntpd[17652]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-07-15T06:33:58.210 INFO:teuthology.orchestra.run.smithi134.stderr:15 Jul 06:33:58 ntpd[17652]: can't open /var/log/ntpstats/rawstats.20240715: Permission denied 2024-07-15T06:33:58.210 INFO:teuthology.orchestra.run.smithi134.stderr:15 Jul 06:33:58 ntpd[17652]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-07-15T06:33:58.210 INFO:teuthology.orchestra.run.smithi134.stderr:15 Jul 06:33:58 ntpd[17652]: can't open /var/log/ntpstats/peerstats.20240715: Permission denied 2024-07-15T06:34:00.185 INFO:teuthology.orchestra.run.smithi089.stderr:15 Jul 06:34:00 ntpd[17654]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-07-15T06:34:00.185 INFO:teuthology.orchestra.run.smithi089.stderr:15 Jul 06:34:00 ntpd[17654]: can't open /var/log/ntpstats/rawstats.20240715: Permission denied 2024-07-15T06:34:00.186 INFO:teuthology.orchestra.run.smithi089.stderr:15 Jul 06:34:00 ntpd[17654]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-07-15T06:34:00.186 INFO:teuthology.orchestra.run.smithi089.stderr:15 Jul 06:34:00 ntpd[17654]: can't open /var/log/ntpstats/rawstats.20240715: Permission denied 2024-07-15T06:34:00.186 INFO:teuthology.orchestra.run.smithi089.stderr:15 Jul 06:34:00 ntpd[17654]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-07-15T06:34:00.186 INFO:teuthology.orchestra.run.smithi089.stderr:15 Jul 06:34:00 ntpd[17654]: can't open /var/log/ntpstats/rawstats.20240715: Permission denied 2024-07-15T06:34:00.186 INFO:teuthology.orchestra.run.smithi089.stderr:15 Jul 06:34:00 ntpd[17654]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-07-15T06:34:00.186 INFO:teuthology.orchestra.run.smithi089.stderr:15 Jul 06:34:00 ntpd[17654]: can't open /var/log/ntpstats/rawstats.20240715: Permission denied 2024-07-15T06:34:00.186 INFO:teuthology.orchestra.run.smithi089.stderr:15 Jul 06:34:00 ntpd[17654]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-07-15T06:34:00.186 INFO:teuthology.orchestra.run.smithi089.stderr:15 Jul 06:34:00 ntpd[17654]: can't open /var/log/ntpstats/peerstats.20240715: Permission denied 2024-07-15T06:34:00.186 INFO:teuthology.orchestra.run.smithi089.stderr:15 Jul 06:34:00 ntpd[17654]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-07-15T06:34:00.186 INFO:teuthology.orchestra.run.smithi089.stderr:15 Jul 06:34:00 ntpd[17654]: can't open /var/log/ntpstats/rawstats.20240715: Permission denied 2024-07-15T06:34:00.186 INFO:teuthology.orchestra.run.smithi089.stderr:15 Jul 06:34:00 ntpd[17654]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-07-15T06:34:00.186 INFO:teuthology.orchestra.run.smithi089.stderr:15 Jul 06:34:00 ntpd[17654]: can't open /var/log/ntpstats/peerstats.20240715: Permission denied 2024-07-15T06:34:00.186 INFO:teuthology.orchestra.run.smithi089.stderr:15 Jul 06:34:00 ntpd[17654]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-07-15T06:34:00.187 INFO:teuthology.orchestra.run.smithi089.stderr:15 Jul 06:34:00 ntpd[17654]: can't open /var/log/ntpstats/rawstats.20240715: Permission denied 2024-07-15T06:34:00.187 INFO:teuthology.orchestra.run.smithi089.stderr:15 Jul 06:34:00 ntpd[17654]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-07-15T06:34:00.187 INFO:teuthology.orchestra.run.smithi089.stderr:15 Jul 06:34:00 ntpd[17654]: can't open /var/log/ntpstats/peerstats.20240715: Permission denied 2024-07-15T06:34:00.209 INFO:teuthology.orchestra.run.smithi134.stderr:15 Jul 06:34:00 ntpd[17652]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-07-15T06:34:00.210 INFO:teuthology.orchestra.run.smithi134.stderr:15 Jul 06:34:00 ntpd[17652]: can't open /var/log/ntpstats/rawstats.20240715: Permission denied 2024-07-15T06:34:00.210 INFO:teuthology.orchestra.run.smithi134.stderr:15 Jul 06:34:00 ntpd[17652]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-07-15T06:34:00.210 INFO:teuthology.orchestra.run.smithi134.stderr:15 Jul 06:34:00 ntpd[17652]: can't open /var/log/ntpstats/rawstats.20240715: Permission denied 2024-07-15T06:34:00.210 INFO:teuthology.orchestra.run.smithi134.stderr:15 Jul 06:34:00 ntpd[17652]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-07-15T06:34:00.210 INFO:teuthology.orchestra.run.smithi134.stderr:15 Jul 06:34:00 ntpd[17652]: can't open /var/log/ntpstats/rawstats.20240715: Permission denied 2024-07-15T06:34:00.210 INFO:teuthology.orchestra.run.smithi134.stderr:15 Jul 06:34:00 ntpd[17652]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-07-15T06:34:00.210 INFO:teuthology.orchestra.run.smithi134.stderr:15 Jul 06:34:00 ntpd[17652]: can't open /var/log/ntpstats/rawstats.20240715: Permission denied 2024-07-15T06:34:00.210 INFO:teuthology.orchestra.run.smithi134.stderr:15 Jul 06:34:00 ntpd[17652]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-07-15T06:34:00.210 INFO:teuthology.orchestra.run.smithi134.stderr:15 Jul 06:34:00 ntpd[17652]: can't open /var/log/ntpstats/peerstats.20240715: Permission denied 2024-07-15T06:34:00.210 INFO:teuthology.orchestra.run.smithi134.stderr:15 Jul 06:34:00 ntpd[17652]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-07-15T06:34:00.210 INFO:teuthology.orchestra.run.smithi134.stderr:15 Jul 06:34:00 ntpd[17652]: can't open /var/log/ntpstats/rawstats.20240715: Permission denied 2024-07-15T06:34:00.211 INFO:teuthology.orchestra.run.smithi134.stderr:15 Jul 06:34:00 ntpd[17652]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-07-15T06:34:00.211 INFO:teuthology.orchestra.run.smithi134.stderr:15 Jul 06:34:00 ntpd[17652]: can't open /var/log/ntpstats/peerstats.20240715: Permission denied 2024-07-15T06:34:00.211 INFO:teuthology.orchestra.run.smithi134.stderr:15 Jul 06:34:00 ntpd[17652]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-07-15T06:34:00.211 INFO:teuthology.orchestra.run.smithi134.stderr:15 Jul 06:34:00 ntpd[17652]: can't open /var/log/ntpstats/rawstats.20240715: Permission denied 2024-07-15T06:34:00.211 INFO:teuthology.orchestra.run.smithi134.stderr:15 Jul 06:34:00 ntpd[17652]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-07-15T06:34:00.211 INFO:teuthology.orchestra.run.smithi134.stderr:15 Jul 06:34:00 ntpd[17652]: can't open /var/log/ntpstats/peerstats.20240715: Permission denied 2024-07-15T06:34:02.185 INFO:teuthology.orchestra.run.smithi089.stderr:15 Jul 06:34:02 ntpd[17654]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-07-15T06:34:02.185 INFO:teuthology.orchestra.run.smithi089.stderr:15 Jul 06:34:02 ntpd[17654]: can't open /var/log/ntpstats/rawstats.20240715: Permission denied 2024-07-15T06:34:02.185 INFO:teuthology.orchestra.run.smithi089.stderr:15 Jul 06:34:02 ntpd[17654]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-07-15T06:34:02.185 INFO:teuthology.orchestra.run.smithi089.stderr:15 Jul 06:34:02 ntpd[17654]: can't open /var/log/ntpstats/rawstats.20240715: Permission denied 2024-07-15T06:34:02.185 INFO:teuthology.orchestra.run.smithi089.stderr:15 Jul 06:34:02 ntpd[17654]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-07-15T06:34:02.185 INFO:teuthology.orchestra.run.smithi089.stderr:15 Jul 06:34:02 ntpd[17654]: can't open /var/log/ntpstats/rawstats.20240715: Permission denied 2024-07-15T06:34:02.185 INFO:teuthology.orchestra.run.smithi089.stderr:15 Jul 06:34:02 ntpd[17654]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-07-15T06:34:02.185 INFO:teuthology.orchestra.run.smithi089.stderr:15 Jul 06:34:02 ntpd[17654]: can't open /var/log/ntpstats/rawstats.20240715: Permission denied 2024-07-15T06:34:02.185 INFO:teuthology.orchestra.run.smithi089.stderr:15 Jul 06:34:02 ntpd[17654]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-07-15T06:34:02.185 INFO:teuthology.orchestra.run.smithi089.stderr:15 Jul 06:34:02 ntpd[17654]: can't open /var/log/ntpstats/peerstats.20240715: Permission denied 2024-07-15T06:34:02.186 INFO:teuthology.orchestra.run.smithi089.stderr:15 Jul 06:34:02 ntpd[17654]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-07-15T06:34:02.186 INFO:teuthology.orchestra.run.smithi089.stderr:15 Jul 06:34:02 ntpd[17654]: can't open /var/log/ntpstats/rawstats.20240715: Permission denied 2024-07-15T06:34:02.186 INFO:teuthology.orchestra.run.smithi089.stderr:15 Jul 06:34:02 ntpd[17654]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-07-15T06:34:02.186 INFO:teuthology.orchestra.run.smithi089.stderr:15 Jul 06:34:02 ntpd[17654]: can't open /var/log/ntpstats/peerstats.20240715: Permission denied 2024-07-15T06:34:02.186 INFO:teuthology.orchestra.run.smithi089.stdout:15 Jul 06:34:02 ntpd[17654]: ntpd: time slew -0.001292 s 2024-07-15T06:34:02.186 INFO:teuthology.orchestra.run.smithi089.stdout:ntpd: time slew -0.001292s 2024-07-15T06:34:02.186 INFO:teuthology.orchestra.run.smithi089.stderr:15 Jul 06:34:02 ntpd[17654]: couldn't unlink /var/log/ntpstats/loopstats: Permission denied 2024-07-15T06:34:02.187 INFO:teuthology.orchestra.run.smithi089.stderr:15 Jul 06:34:02 ntpd[17654]: can't open /var/log/ntpstats/loopstats.20240715: Permission denied 2024-07-15T06:34:02.208 INFO:teuthology.orchestra.run.smithi134.stderr:15 Jul 06:34:02 ntpd[17652]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-07-15T06:34:02.209 INFO:teuthology.orchestra.run.smithi134.stderr:15 Jul 06:34:02 ntpd[17652]: can't open /var/log/ntpstats/rawstats.20240715: Permission denied 2024-07-15T06:34:02.209 INFO:teuthology.orchestra.run.smithi134.stderr:15 Jul 06:34:02 ntpd[17652]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-07-15T06:34:02.209 INFO:teuthology.orchestra.run.smithi134.stderr:15 Jul 06:34:02 ntpd[17652]: can't open /var/log/ntpstats/rawstats.20240715: Permission denied 2024-07-15T06:34:02.209 INFO:teuthology.orchestra.run.smithi134.stderr:15 Jul 06:34:02 ntpd[17652]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-07-15T06:34:02.209 INFO:teuthology.orchestra.run.smithi134.stderr:15 Jul 06:34:02 ntpd[17652]: can't open /var/log/ntpstats/rawstats.20240715: Permission denied 2024-07-15T06:34:02.209 INFO:teuthology.orchestra.run.smithi134.stderr:15 Jul 06:34:02 ntpd[17652]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-07-15T06:34:02.209 INFO:teuthology.orchestra.run.smithi134.stderr:15 Jul 06:34:02 ntpd[17652]: can't open /var/log/ntpstats/rawstats.20240715: Permission denied 2024-07-15T06:34:02.209 INFO:teuthology.orchestra.run.smithi134.stderr:15 Jul 06:34:02 ntpd[17652]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-07-15T06:34:02.209 INFO:teuthology.orchestra.run.smithi134.stderr:15 Jul 06:34:02 ntpd[17652]: can't open /var/log/ntpstats/peerstats.20240715: Permission denied 2024-07-15T06:34:02.209 INFO:teuthology.orchestra.run.smithi134.stdout:15 Jul 06:34:02 ntpd[17652]: ntpd: time slew -0.001378 s 2024-07-15T06:34:02.209 INFO:teuthology.orchestra.run.smithi134.stdout:ntpd: time slew -0.001378s 2024-07-15T06:34:02.210 INFO:teuthology.orchestra.run.smithi134.stderr:15 Jul 06:34:02 ntpd[17652]: couldn't unlink /var/log/ntpstats/loopstats: Permission denied 2024-07-15T06:34:02.210 INFO:teuthology.orchestra.run.smithi134.stderr:15 Jul 06:34:02 ntpd[17652]: can't open /var/log/ntpstats/loopstats.20240715: Permission denied 2024-07-15T06:34:02.233 INFO:teuthology.orchestra.run.smithi089.stdout: remote refid st t when poll reach delay offset jitter 2024-07-15T06:34:02.233 INFO:teuthology.orchestra.run.smithi089.stdout:============================================================================== 2024-07-15T06:34:02.233 INFO:teuthology.orchestra.run.smithi089.stdout: hv01.front.sepi .INIT. 16 u - 64 0 0.000 +0.000 0.000 2024-07-15T06:34:02.233 INFO:teuthology.orchestra.run.smithi089.stdout: hv02.front.sepi .INIT. 16 u - 64 0 0.000 +0.000 0.000 2024-07-15T06:34:02.233 INFO:teuthology.orchestra.run.smithi089.stdout: hv03.front.sepi .INIT. 16 u - 64 0 0.000 +0.000 0.000 2024-07-15T06:34:02.233 INFO:teuthology.orchestra.run.smithi089.stdout: hv04.front.sepi .INIT. 16 u - 64 0 0.000 +0.000 0.000 2024-07-15T06:34:02.327 INFO:teuthology.orchestra.run.smithi134.stdout: remote refid st t when poll reach delay offset jitter 2024-07-15T06:34:02.327 INFO:teuthology.orchestra.run.smithi134.stdout:============================================================================== 2024-07-15T06:34:02.327 INFO:teuthology.orchestra.run.smithi134.stdout: hv01.front.sepi .INIT. 16 u - 64 0 0.000 +0.000 0.000 2024-07-15T06:34:02.327 INFO:teuthology.orchestra.run.smithi134.stdout: hv02.front.sepi .INIT. 16 u - 64 0 0.000 +0.000 0.000 2024-07-15T06:34:02.327 INFO:teuthology.orchestra.run.smithi134.stdout: hv03.front.sepi .INIT. 16 u - 64 0 0.000 +0.000 0.000 2024-07-15T06:34:02.327 INFO:teuthology.orchestra.run.smithi134.stdout: hv04.front.sepi .INIT. 16 u - 64 0 0.000 +0.000 0.000 2024-07-15T06:34:02.328 INFO:teuthology.run_tasks:Running task install... 2024-07-15T06:34:02.336 DEBUG:teuthology.task.install:project ceph 2024-07-15T06:34:02.336 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': '6ea7aaa9f93d34dc4d21862afe00140dfddbd706'}} 2024-07-15T06:34:02.336 DEBUG:teuthology.task.install:config {'branch': 'quincy', 'exclude_packages': ['ceph-volume'], 'flavor': 'default', 'sha1': '6ea7aaa9f93d34dc4d21862afe00140dfddbd706'} 2024-07-15T06:34:02.336 INFO:teuthology.task.install:Using flavor: default 2024-07-15T06:34:02.341 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2024-07-15T06:34:02.341 INFO:teuthology.task.install:extra packages: [] 2024-07-15T06:34:02.342 DEBUG:teuthology.orchestra.run.smithi089:> sudo apt-key list | grep Ceph 2024-07-15T06:34:02.342 DEBUG:teuthology.orchestra.run.smithi134:> sudo apt-key list | grep Ceph 2024-07-15T06:34:02.401 INFO:teuthology.orchestra.run.smithi089.stderr:Warning: apt-key is deprecated. Manage keyring files in trusted.gpg.d instead (see apt-key(8)). 2024-07-15T06:34:02.428 INFO:teuthology.orchestra.run.smithi089.stdout:uid [ unknown] Ceph automated package build (Ceph automated package build) 2024-07-15T06:34:02.428 INFO:teuthology.orchestra.run.smithi089.stdout:uid [ unknown] Ceph.com (release key) 2024-07-15T06:34:02.429 INFO:teuthology.task.install.deb:Installing packages: ceph, cephadm, ceph-mds, ceph-mgr, ceph-common, ceph-fuse, ceph-test, radosgw, python3-rados, python3-rgw, python3-cephfs, python3-rbd, libcephfs2, libcephfs-dev, librados2, librbd1, rbd-fuse on remote deb x86_64 2024-07-15T06:34:02.429 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-07-15T06:34:02.429 INFO:teuthology.packaging:ref: None 2024-07-15T06:34:02.430 INFO:teuthology.packaging:tag: None 2024-07-15T06:34:02.430 INFO:teuthology.packaging:branch: quincy 2024-07-15T06:34:02.430 INFO:teuthology.packaging:sha1: 6ea7aaa9f93d34dc4d21862afe00140dfddbd706 2024-07-15T06:34:02.430 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&ref=quincy 2024-07-15T06:34:02.435 INFO:teuthology.orchestra.run.smithi134.stderr:Warning: apt-key is deprecated. Manage keyring files in trusted.gpg.d instead (see apt-key(8)). 2024-07-15T06:34:02.464 INFO:teuthology.orchestra.run.smithi134.stdout:uid [ unknown] Ceph automated package build (Ceph automated package build) 2024-07-15T06:34:02.464 INFO:teuthology.orchestra.run.smithi134.stdout:uid [ unknown] Ceph.com (release key) 2024-07-15T06:34:02.464 INFO:teuthology.task.install.deb:Installing packages: ceph, cephadm, ceph-mds, ceph-mgr, ceph-common, ceph-fuse, ceph-test, radosgw, python3-rados, python3-rgw, python3-cephfs, python3-rbd, libcephfs2, libcephfs-dev, librados2, librbd1, rbd-fuse on remote deb x86_64 2024-07-15T06:34:02.465 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-07-15T06:34:02.465 INFO:teuthology.packaging:ref: None 2024-07-15T06:34:02.465 INFO:teuthology.packaging:tag: None 2024-07-15T06:34:02.465 INFO:teuthology.packaging:branch: quincy 2024-07-15T06:34:02.465 INFO:teuthology.packaging:sha1: 6ea7aaa9f93d34dc4d21862afe00140dfddbd706 2024-07-15T06:34:02.465 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&ref=quincy 2024-07-15T06:34:02.579 INFO:teuthology.task.install.deb:Pulling from https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default/ 2024-07-15T06:34:02.580 INFO:teuthology.task.install.deb:Package version is 17.2.7-1jammy 2024-07-15T06:34:02.603 INFO:teuthology.task.install.deb:Pulling from https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default/ 2024-07-15T06:34:02.604 INFO:teuthology.task.install.deb:Package version is 17.2.7-1jammy 2024-07-15T06:34:02.616 DEBUG:teuthology.orchestra.run.smithi089:> set -ex 2024-07-15T06:34:02.616 DEBUG:teuthology.orchestra.run.smithi089:> sudo dd of=/etc/apt/sources.list.d/ceph.list 2024-07-15T06:34:02.631 DEBUG:teuthology.orchestra.run.smithi089:> sudo apt-get update 2024-07-15T06:34:02.643 DEBUG:teuthology.orchestra.run.smithi134:> set -ex 2024-07-15T06:34:02.643 DEBUG:teuthology.orchestra.run.smithi134:> sudo dd of=/etc/apt/sources.list.d/ceph.list 2024-07-15T06:34:02.657 DEBUG:teuthology.orchestra.run.smithi134:> sudo apt-get update 2024-07-15T06:34:02.771 INFO:teuthology.orchestra.run.smithi089.stdout:Ign:1 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy InRelease 2024-07-15T06:34:02.772 INFO:teuthology.orchestra.run.smithi089.stdout:Get:2 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy Release [7,641 B] 2024-07-15T06:34:02.774 INFO:teuthology.orchestra.run.smithi089.stdout:Ign:3 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy Release.gpg 2024-07-15T06:34:02.774 INFO:teuthology.orchestra.run.smithi089.stdout:Get:4 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 Packages [17.3 kB] 2024-07-15T06:34:02.783 INFO:teuthology.orchestra.run.smithi089.stdout:Get:5 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main i386 Packages [3,331 B] 2024-07-15T06:34:02.801 INFO:teuthology.orchestra.run.smithi134.stdout:Ign:1 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy InRelease 2024-07-15T06:34:02.802 INFO:teuthology.orchestra.run.smithi134.stdout:Get:2 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy Release [7,641 B] 2024-07-15T06:34:02.804 INFO:teuthology.orchestra.run.smithi134.stdout:Ign:3 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy Release.gpg 2024-07-15T06:34:02.805 INFO:teuthology.orchestra.run.smithi134.stdout:Get:4 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 Packages [17.3 kB] 2024-07-15T06:34:02.812 INFO:teuthology.orchestra.run.smithi134.stdout:Get:5 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main i386 Packages [3,331 B] 2024-07-15T06:34:02.827 INFO:teuthology.orchestra.run.smithi134.stdout:Hit:6 http://archive.ubuntu.com/ubuntu jammy InRelease 2024-07-15T06:34:02.838 INFO:teuthology.orchestra.run.smithi134.stdout:Hit:7 http://archive.ubuntu.com/ubuntu jammy-updates InRelease 2024-07-15T06:34:02.855 INFO:teuthology.orchestra.run.smithi134.stdout:Hit:8 http://archive.ubuntu.com/ubuntu jammy-backports InRelease 2024-07-15T06:34:02.928 INFO:teuthology.orchestra.run.smithi089.stdout:Hit:6 http://security.ubuntu.com/ubuntu jammy-security InRelease 2024-07-15T06:34:02.949 INFO:teuthology.orchestra.run.smithi134.stdout:Hit:9 http://security.ubuntu.com/ubuntu jammy-security InRelease 2024-07-15T06:34:03.179 INFO:teuthology.orchestra.run.smithi134.stdout:Fetched 28.3 kB in 0s (83.5 kB/s) 2024-07-15T06:34:04.181 INFO:teuthology.orchestra.run.smithi134.stdout:Reading package lists... 2024-07-15T06:34:04.197 DEBUG:teuthology.orchestra.run.smithi134:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install ceph=17.2.7-1jammy cephadm=17.2.7-1jammy ceph-mds=17.2.7-1jammy ceph-mgr=17.2.7-1jammy ceph-common=17.2.7-1jammy ceph-fuse=17.2.7-1jammy ceph-test=17.2.7-1jammy radosgw=17.2.7-1jammy python3-rados=17.2.7-1jammy python3-rgw=17.2.7-1jammy python3-cephfs=17.2.7-1jammy python3-rbd=17.2.7-1jammy libcephfs2=17.2.7-1jammy libcephfs-dev=17.2.7-1jammy librados2=17.2.7-1jammy librbd1=17.2.7-1jammy rbd-fuse=17.2.7-1jammy 2024-07-15T06:34:04.259 INFO:teuthology.orchestra.run.smithi134.stdout:Reading package lists... 2024-07-15T06:34:04.442 INFO:teuthology.orchestra.run.smithi134.stdout:Building dependency tree... 2024-07-15T06:34:04.443 INFO:teuthology.orchestra.run.smithi134.stdout:Reading state information... 2024-07-15T06:34:04.619 INFO:teuthology.orchestra.run.smithi134.stdout:The following packages were automatically installed and are no longer required: 2024-07-15T06:34:04.619 INFO:teuthology.orchestra.run.smithi134.stdout: libboost-iostreams1.74.0 libboost-thread1.74.0 libgfapi0 libgfrpc0 libgfxdr0 2024-07-15T06:34:04.619 INFO:teuthology.orchestra.run.smithi134.stdout: libglusterfs0 libiscsi7 libpmemobj1 libpython2-dev libpython2-stdlib 2024-07-15T06:34:04.619 INFO:teuthology.orchestra.run.smithi134.stdout: libpython2.7 libpython2.7-dev libpython2.7-minimal libpython2.7-stdlib 2024-07-15T06:34:04.619 INFO:teuthology.orchestra.run.smithi134.stdout: linux-image-5.4.0-135-generic linux-modules-5.4.0-135-generic python2 2024-07-15T06:34:04.619 INFO:teuthology.orchestra.run.smithi134.stdout: python2-dev python2-minimal python2.7 python2.7-dev python2.7-minimal 2024-07-15T06:34:04.619 INFO:teuthology.orchestra.run.smithi134.stdout:Use 'sudo apt autoremove' to remove them. 2024-07-15T06:34:04.620 INFO:teuthology.orchestra.run.smithi134.stdout:The following additional packages will be installed: 2024-07-15T06:34:04.620 INFO:teuthology.orchestra.run.smithi134.stdout: ceph-base ceph-mgr-cephadm ceph-mgr-dashboard ceph-mgr-diskprediction-local 2024-07-15T06:34:04.620 INFO:teuthology.orchestra.run.smithi134.stdout: ceph-mgr-k8sevents ceph-mgr-modules-core ceph-mon ceph-osd ceph-volume jq 2024-07-15T06:34:04.620 INFO:teuthology.orchestra.run.smithi134.stdout: libdouble-conversion3 libfmt8 libjq1 liblttng-ust1 liboath0 libonig5 2024-07-15T06:34:04.620 INFO:teuthology.orchestra.run.smithi134.stdout: libpcre2-16-0 libqt5core5a libqt5dbus5 libqt5network5 libradosstriper1 2024-07-15T06:34:04.621 INFO:teuthology.orchestra.run.smithi134.stdout: librdkafka1 librgw2 libsqlite3-mod-ceph libthrift-0.16.0 nvme-cli 2024-07-15T06:34:04.621 INFO:teuthology.orchestra.run.smithi134.stdout: python-babel-localedata python-pastedeploy-tpl python3-babel python3-bcrypt 2024-07-15T06:34:04.621 INFO:teuthology.orchestra.run.smithi134.stdout: python3-cachetools python3-ceph-argparse python3-ceph-common python3-certifi 2024-07-15T06:34:04.621 INFO:teuthology.orchestra.run.smithi134.stdout: python3-cheroot python3-cherrypy3 python3-google-auth python3-idna 2024-07-15T06:34:04.621 INFO:teuthology.orchestra.run.smithi134.stdout: python3-iniconfig python3-jaraco.classes python3-jaraco.collections 2024-07-15T06:34:04.622 INFO:teuthology.orchestra.run.smithi134.stdout: python3-jaraco.functools python3-jaraco.text python3-jinja2 python3-joblib 2024-07-15T06:34:04.622 INFO:teuthology.orchestra.run.smithi134.stdout: python3-kubernetes python3-logutils python3-mako python3-markupsafe 2024-07-15T06:34:04.622 INFO:teuthology.orchestra.run.smithi134.stdout: python3-natsort python3-openssl python3-paste python3-pastedeploy 2024-07-15T06:34:04.622 INFO:teuthology.orchestra.run.smithi134.stdout: python3-pastescript python3-pecan python3-pluggy python3-portend 2024-07-15T06:34:04.622 INFO:teuthology.orchestra.run.smithi134.stdout: python3-prettytable python3-psutil python3-py python3-pyasn1 2024-07-15T06:34:04.622 INFO:teuthology.orchestra.run.smithi134.stdout: python3-pyasn1-modules python3-pygments python3-pyinotify python3-pytest 2024-07-15T06:34:04.622 INFO:teuthology.orchestra.run.smithi134.stdout: python3-repoze.lru python3-requests python3-requests-oauthlib python3-routes 2024-07-15T06:34:04.622 INFO:teuthology.orchestra.run.smithi134.stdout: python3-rsa python3-simplegeneric python3-simplejson python3-singledispatch 2024-07-15T06:34:04.622 INFO:teuthology.orchestra.run.smithi134.stdout: python3-sklearn python3-sklearn-lib python3-tempita python3-tempora 2024-07-15T06:34:04.622 INFO:teuthology.orchestra.run.smithi134.stdout: python3-threadpoolctl python3-toml python3-urllib3 python3-waitress 2024-07-15T06:34:04.622 INFO:teuthology.orchestra.run.smithi134.stdout: python3-wcwidth python3-webob python3-websocket python3-webtest 2024-07-15T06:34:04.622 INFO:teuthology.orchestra.run.smithi134.stdout: python3-werkzeug python3-zc.lockfile qttranslations5-l10n socat xmlstarlet 2024-07-15T06:34:04.623 INFO:teuthology.orchestra.run.smithi134.stdout:Suggested packages: 2024-07-15T06:34:04.623 INFO:teuthology.orchestra.run.smithi134.stdout: python3-influxdb python-jinja2-doc python3-beaker python-mako-doc 2024-07-15T06:34:04.623 INFO:teuthology.orchestra.run.smithi134.stdout: python-natsort-doc python-openssl-doc python3-openssl-dbg httpd-wsgi 2024-07-15T06:34:04.623 INFO:teuthology.orchestra.run.smithi134.stdout: libapache2-mod-python libapache2-mod-scgi libjs-mochikit python-pecan-doc 2024-07-15T06:34:04.623 INFO:teuthology.orchestra.run.smithi134.stdout: python-psutil-doc subversion python-pygments-doc python-pyinotify-doc 2024-07-15T06:34:04.623 INFO:teuthology.orchestra.run.smithi134.stdout: python3-socks python-requests-doc python3-dap python-sklearn-doc ipython3 2024-07-15T06:34:04.623 INFO:teuthology.orchestra.run.smithi134.stdout: python-waitress-doc python-webob-doc python-webtest-doc python-werkzeug-doc 2024-07-15T06:34:04.623 INFO:teuthology.orchestra.run.smithi134.stdout: python3-watchdog 2024-07-15T06:34:04.623 INFO:teuthology.orchestra.run.smithi134.stdout:Recommended packages: 2024-07-15T06:34:04.623 INFO:teuthology.orchestra.run.smithi134.stdout: btrfs-tools 2024-07-15T06:34:04.722 INFO:teuthology.orchestra.run.smithi134.stdout:The following NEW packages will be installed: 2024-07-15T06:34:04.722 INFO:teuthology.orchestra.run.smithi134.stdout: ceph ceph-base ceph-common ceph-fuse ceph-mds ceph-mgr ceph-mgr-cephadm 2024-07-15T06:34:04.722 INFO:teuthology.orchestra.run.smithi134.stdout: ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-k8sevents 2024-07-15T06:34:04.722 INFO:teuthology.orchestra.run.smithi134.stdout: ceph-mgr-modules-core ceph-mon ceph-osd ceph-test ceph-volume cephadm jq 2024-07-15T06:34:04.722 INFO:teuthology.orchestra.run.smithi134.stdout: libcephfs-dev libcephfs2 libdouble-conversion3 libfmt8 libjq1 liblttng-ust1 2024-07-15T06:34:04.722 INFO:teuthology.orchestra.run.smithi134.stdout: liboath0 libonig5 libpcre2-16-0 libqt5core5a libqt5dbus5 libqt5network5 2024-07-15T06:34:04.722 INFO:teuthology.orchestra.run.smithi134.stdout: librados2 libradosstriper1 librbd1 librdkafka1 librgw2 libsqlite3-mod-ceph 2024-07-15T06:34:04.723 INFO:teuthology.orchestra.run.smithi134.stdout: libthrift-0.16.0 nvme-cli python-babel-localedata python-pastedeploy-tpl 2024-07-15T06:34:04.723 INFO:teuthology.orchestra.run.smithi134.stdout: python3-babel python3-bcrypt python3-cachetools python3-ceph-argparse 2024-07-15T06:34:04.723 INFO:teuthology.orchestra.run.smithi134.stdout: python3-ceph-common python3-cephfs python3-certifi python3-cheroot 2024-07-15T06:34:04.723 INFO:teuthology.orchestra.run.smithi134.stdout: python3-cherrypy3 python3-google-auth python3-idna python3-iniconfig 2024-07-15T06:34:04.723 INFO:teuthology.orchestra.run.smithi134.stdout: python3-jaraco.classes python3-jaraco.collections python3-jaraco.functools 2024-07-15T06:34:04.723 INFO:teuthology.orchestra.run.smithi134.stdout: python3-jaraco.text python3-jinja2 python3-joblib python3-kubernetes 2024-07-15T06:34:04.724 INFO:teuthology.orchestra.run.smithi134.stdout: python3-logutils python3-mako python3-markupsafe python3-natsort 2024-07-15T06:34:04.724 INFO:teuthology.orchestra.run.smithi134.stdout: python3-openssl python3-paste python3-pastedeploy python3-pastescript 2024-07-15T06:34:04.724 INFO:teuthology.orchestra.run.smithi134.stdout: python3-pecan python3-pluggy python3-portend python3-prettytable 2024-07-15T06:34:04.724 INFO:teuthology.orchestra.run.smithi134.stdout: python3-psutil python3-py python3-pyasn1 python3-pyasn1-modules 2024-07-15T06:34:04.724 INFO:teuthology.orchestra.run.smithi134.stdout: python3-pygments python3-pyinotify python3-pytest python3-rados python3-rbd 2024-07-15T06:34:04.724 INFO:teuthology.orchestra.run.smithi134.stdout: python3-repoze.lru python3-requests python3-requests-oauthlib python3-rgw 2024-07-15T06:34:04.724 INFO:teuthology.orchestra.run.smithi134.stdout: python3-routes python3-rsa python3-simplegeneric python3-simplejson 2024-07-15T06:34:04.724 INFO:teuthology.orchestra.run.smithi134.stdout: python3-singledispatch python3-sklearn python3-sklearn-lib python3-tempita 2024-07-15T06:34:04.724 INFO:teuthology.orchestra.run.smithi134.stdout: python3-tempora python3-threadpoolctl python3-toml python3-urllib3 2024-07-15T06:34:04.724 INFO:teuthology.orchestra.run.smithi134.stdout: python3-waitress python3-wcwidth python3-webob python3-websocket 2024-07-15T06:34:04.724 INFO:teuthology.orchestra.run.smithi134.stdout: python3-webtest python3-werkzeug python3-zc.lockfile qttranslations5-l10n 2024-07-15T06:34:04.724 INFO:teuthology.orchestra.run.smithi134.stdout: radosgw rbd-fuse socat xmlstarlet 2024-07-15T06:34:04.765 INFO:teuthology.orchestra.run.smithi134.stdout:0 upgraded, 107 newly installed, 0 to remove and 316 not upgraded. 2024-07-15T06:34:04.765 INFO:teuthology.orchestra.run.smithi134.stdout:Need to get 187 MB of archives. 2024-07-15T06:34:04.765 INFO:teuthology.orchestra.run.smithi134.stdout:After this operation, 815 MB of additional disk space will be used. 2024-07-15T06:34:04.765 INFO:teuthology.orchestra.run.smithi134.stdout:Get:1 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 librados2 amd64 17.2.7-1jammy [3,198 kB] 2024-07-15T06:34:04.794 INFO:teuthology.orchestra.run.smithi134.stdout:Get:2 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 librbd1 amd64 17.2.7-1jammy [3,067 kB] 2024-07-15T06:34:04.817 INFO:teuthology.orchestra.run.smithi134.stdout:Get:3 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 libcephfs2 amd64 17.2.7-1jammy [632 kB] 2024-07-15T06:34:04.822 INFO:teuthology.orchestra.run.smithi134.stdout:Get:4 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 python3-rados amd64 17.2.7-1jammy [344 kB] 2024-07-15T06:34:04.824 INFO:teuthology.orchestra.run.smithi134.stdout:Get:5 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 python3-ceph-argparse all 17.2.7-1jammy [23.1 kB] 2024-07-15T06:34:04.825 INFO:teuthology.orchestra.run.smithi134.stdout:Get:6 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 python3-cephfs amd64 17.2.7-1jammy [164 kB] 2024-07-15T06:34:04.826 INFO:teuthology.orchestra.run.smithi134.stdout:Get:7 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 python3-ceph-common all 17.2.7-1jammy [47.7 kB] 2024-07-15T06:34:04.826 INFO:teuthology.orchestra.run.smithi134.stdout:Get:8 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 python3-rbd amd64 17.2.7-1jammy [313 kB] 2024-07-15T06:34:04.829 INFO:teuthology.orchestra.run.smithi134.stdout:Get:9 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 librgw2 amd64 17.2.7-1jammy [5,086 kB] 2024-07-15T06:34:04.862 INFO:teuthology.orchestra.run.smithi134.stdout:Get:10 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 python3-rgw amd64 17.2.7-1jammy [102 kB] 2024-07-15T06:34:04.863 INFO:teuthology.orchestra.run.smithi134.stdout:Get:11 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 libradosstriper1 amd64 17.2.7-1jammy [415 kB] 2024-07-15T06:34:04.866 INFO:teuthology.orchestra.run.smithi134.stdout:Get:12 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-common amd64 17.2.7-1jammy [25.3 MB] 2024-07-15T06:34:04.912 INFO:teuthology.orchestra.run.smithi134.stdout:Get:13 http://archive.ubuntu.com/ubuntu jammy/universe amd64 libfmt8 amd64 8.1.1+ds1-2 [60.2 kB] 2024-07-15T06:34:05.130 INFO:teuthology.orchestra.run.smithi134.stdout:Get:14 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-base amd64 17.2.7-1jammy [6,378 kB] 2024-07-15T06:34:05.174 INFO:teuthology.orchestra.run.smithi134.stdout:Get:15 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-modules-core all 17.2.7-1jammy [224 kB] 2024-07-15T06:34:05.175 INFO:teuthology.orchestra.run.smithi134.stdout:Get:16 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 libsqlite3-mod-ceph amd64 17.2.7-1jammy [115 kB] 2024-07-15T06:34:05.176 INFO:teuthology.orchestra.run.smithi134.stdout:Get:17 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr amd64 17.2.7-1jammy [1,429 kB] 2024-07-15T06:34:05.186 INFO:teuthology.orchestra.run.smithi134.stdout:Get:18 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mon amd64 17.2.7-1jammy [8,201 kB] 2024-07-15T06:34:05.212 INFO:teuthology.orchestra.run.smithi134.stdout:Get:19 http://archive.ubuntu.com/ubuntu jammy/main amd64 liblttng-ust1 amd64 2.13.1-1ubuntu1 [190 kB] 2024-07-15T06:34:05.241 INFO:teuthology.orchestra.run.smithi134.stdout:Get:20 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-osd amd64 17.2.7-1jammy [28.4 MB] 2024-07-15T06:34:05.385 INFO:teuthology.orchestra.run.smithi134.stdout:Get:21 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-wcwidth all 0.2.5+dfsg1-1 [21.9 kB] 2024-07-15T06:34:05.393 INFO:teuthology.orchestra.run.smithi134.stdout:Get:22 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-prettytable all 2.5.0-2 [31.3 kB] 2024-07-15T06:34:05.405 INFO:teuthology.orchestra.run.smithi134.stdout:Get:23 http://archive.ubuntu.com/ubuntu jammy/universe amd64 librdkafka1 amd64 1.8.0-1build1 [633 kB] 2024-07-15T06:34:05.512 INFO:teuthology.orchestra.run.smithi134.stdout:Get:24 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph amd64 17.2.7-1jammy [3,690 B] 2024-07-15T06:34:05.584 INFO:teuthology.orchestra.run.smithi134.stdout:Get:25 http://archive.ubuntu.com/ubuntu jammy/universe amd64 libdouble-conversion3 amd64 3.1.7-4 [39.0 kB] 2024-07-15T06:34:05.584 INFO:teuthology.orchestra.run.smithi134.stdout:Get:26 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-fuse amd64 17.2.7-1jammy [783 kB] 2024-07-15T06:34:05.585 INFO:teuthology.orchestra.run.smithi134.stdout:Get:27 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libpcre2-16-0 amd64 10.39-3ubuntu0.1 [203 kB] 2024-07-15T06:34:05.590 INFO:teuthology.orchestra.run.smithi134.stdout:Get:28 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mds amd64 17.2.7-1jammy [2,120 kB] 2024-07-15T06:34:05.590 INFO:teuthology.orchestra.run.smithi134.stdout:Get:29 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5core5a amd64 5.15.3+dfsg-2ubuntu0.2 [2,006 kB] 2024-07-15T06:34:05.605 INFO:teuthology.orchestra.run.smithi134.stdout:Get:30 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 cephadm amd64 17.2.7-1jammy [80.2 kB] 2024-07-15T06:34:05.605 INFO:teuthology.orchestra.run.smithi134.stdout:Get:31 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-cephadm all 17.2.7-1jammy [118 kB] 2024-07-15T06:34:05.606 INFO:teuthology.orchestra.run.smithi134.stdout:Get:32 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-dashboard all 17.2.7-1jammy [1,824 kB] 2024-07-15T06:34:05.619 INFO:teuthology.orchestra.run.smithi134.stdout:Get:33 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-diskprediction-local all 17.2.7-1jammy [8,615 kB] 2024-07-15T06:34:05.680 INFO:teuthology.orchestra.run.smithi134.stdout:Get:34 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-k8sevents all 17.2.7-1jammy [3,814 B] 2024-07-15T06:34:05.680 INFO:teuthology.orchestra.run.smithi134.stdout:Get:35 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-test amd64 17.2.7-1jammy [52.7 MB] 2024-07-15T06:34:05.694 INFO:teuthology.orchestra.run.smithi134.stdout:Get:36 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5dbus5 amd64 5.15.3+dfsg-2ubuntu0.2 [222 kB] 2024-07-15T06:34:05.702 INFO:teuthology.orchestra.run.smithi134.stdout:Get:37 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5network5 amd64 5.15.3+dfsg-2ubuntu0.2 [731 kB] 2024-07-15T06:34:05.725 INFO:teuthology.orchestra.run.smithi134.stdout:Get:38 http://archive.ubuntu.com/ubuntu jammy/universe amd64 libthrift-0.16.0 amd64 0.16.0-2 [267 kB] 2024-07-15T06:34:05.732 INFO:teuthology.orchestra.run.smithi134.stdout:Get:39 http://archive.ubuntu.com/ubuntu jammy/main amd64 liboath0 amd64 2.6.7-3build1 [41.1 kB] 2024-07-15T06:34:05.734 INFO:teuthology.orchestra.run.smithi134.stdout:Get:40 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-openssl all 21.0.0-1 [45.2 kB] 2024-07-15T06:34:05.735 INFO:teuthology.orchestra.run.smithi134.stdout:Get:41 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-bcrypt amd64 3.2.0-1build1 [32.7 kB] 2024-07-15T06:34:05.736 INFO:teuthology.orchestra.run.smithi134.stdout:Get:42 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.functools all 3.4.0-2 [9,030 B] 2024-07-15T06:34:05.750 INFO:teuthology.orchestra.run.smithi134.stdout:Get:43 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-cheroot all 8.5.2+ds1-1ubuntu3.1 [71.1 kB] 2024-07-15T06:34:05.752 INFO:teuthology.orchestra.run.smithi134.stdout:Get:44 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.classes all 3.2.1-3 [6,452 B] 2024-07-15T06:34:05.832 INFO:teuthology.orchestra.run.smithi134.stdout:Get:45 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.text all 3.6.0-2 [8,716 B] 2024-07-15T06:34:05.832 INFO:teuthology.orchestra.run.smithi134.stdout:Get:46 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.collections all 3.4.0-2 [11.4 kB] 2024-07-15T06:34:05.833 INFO:teuthology.orchestra.run.smithi134.stdout:Get:47 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-tempora all 4.1.2-1 [14.8 kB] 2024-07-15T06:34:05.834 INFO:teuthology.orchestra.run.smithi134.stdout:Get:48 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-portend all 3.0.0-1 [7,240 B] 2024-07-15T06:34:05.834 INFO:teuthology.orchestra.run.smithi134.stdout:Get:49 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-zc.lockfile all 2.0-1 [8,980 B] 2024-07-15T06:34:05.835 INFO:teuthology.orchestra.run.smithi134.stdout:Get:50 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-cherrypy3 all 18.6.1-4 [208 kB] 2024-07-15T06:34:05.840 INFO:teuthology.orchestra.run.smithi134.stdout:Get:51 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-natsort all 8.0.2-1 [35.3 kB] 2024-07-15T06:34:05.842 INFO:teuthology.orchestra.run.smithi134.stdout:Get:52 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-logutils all 0.3.3-8 [17.6 kB] 2024-07-15T06:34:05.913 INFO:teuthology.orchestra.run.smithi134.stdout:Get:53 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-markupsafe amd64 2.0.1-2build1 [12.7 kB] 2024-07-15T06:34:05.914 INFO:teuthology.orchestra.run.smithi134.stdout:Get:54 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-mako all 1.1.3+ds1-2ubuntu0.1 [60.5 kB] 2024-07-15T06:34:05.995 INFO:teuthology.orchestra.run.smithi134.stdout:Get:55 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-simplegeneric all 0.8.1-3 [11.3 kB] 2024-07-15T06:34:05.995 INFO:teuthology.orchestra.run.smithi134.stdout:Get:56 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-singledispatch all 3.4.0.3-3 [7,320 B] 2024-07-15T06:34:05.996 INFO:teuthology.orchestra.run.smithi134.stdout:Get:57 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-webob all 1:1.8.6-1.1 [85.2 kB] 2024-07-15T06:34:05.998 INFO:teuthology.orchestra.run.smithi134.stdout:Get:58 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-waitress all 1.4.4-1.1ubuntu1 [47.3 kB] 2024-07-15T06:34:06.000 INFO:teuthology.orchestra.run.smithi134.stdout:Get:59 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-tempita all 0.5.2-6ubuntu1 [15.1 kB] 2024-07-15T06:34:06.000 INFO:teuthology.orchestra.run.smithi134.stdout:Get:60 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-paste all 3.5.0+dfsg1-1 [456 kB] 2024-07-15T06:34:06.011 INFO:teuthology.orchestra.run.smithi134.stdout:Get:61 http://archive.ubuntu.com/ubuntu jammy/main amd64 python-pastedeploy-tpl all 2.1.1-1 [4,892 B] 2024-07-15T06:34:06.011 INFO:teuthology.orchestra.run.smithi134.stdout:Get:62 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pastedeploy all 2.1.1-1 [26.6 kB] 2024-07-15T06:34:06.076 INFO:teuthology.orchestra.run.smithi134.stdout:Get:63 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-webtest all 2.0.35-1 [28.5 kB] 2024-07-15T06:34:06.115 INFO:teuthology.orchestra.run.smithi134.stdout:Get:64 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pecan all 1.3.3-4ubuntu2 [87.3 kB] 2024-07-15T06:34:06.183 INFO:teuthology.orchestra.run.smithi134.stdout:Get:65 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-certifi all 2020.6.20-1 [150 kB] 2024-07-15T06:34:06.253 INFO:teuthology.orchestra.run.smithi134.stdout:Get:66 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-idna all 3.3-1ubuntu0.1 [52.1 kB] 2024-07-15T06:34:06.255 INFO:teuthology.orchestra.run.smithi134.stdout:Get:67 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-volume all 17.2.7-1jammy [240 kB] 2024-07-15T06:34:06.255 INFO:teuthology.orchestra.run.smithi134.stdout:Get:68 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-urllib3 all 1.26.5-1~exp1ubuntu0.1 [98.2 kB] 2024-07-15T06:34:06.257 INFO:teuthology.orchestra.run.smithi134.stdout:Get:69 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 libcephfs-dev amd64 17.2.7-1jammy [22.4 kB] 2024-07-15T06:34:06.258 INFO:teuthology.orchestra.run.smithi134.stdout:Get:70 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 radosgw amd64 17.2.7-1jammy [15.4 MB] 2024-07-15T06:34:06.258 INFO:teuthology.orchestra.run.smithi134.stdout:Get:71 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-requests all 2.25.1+dfsg-2ubuntu0.1 [48.8 kB] 2024-07-15T06:34:06.259 INFO:teuthology.orchestra.run.smithi134.stdout:Get:72 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-werkzeug all 2.0.2+dfsg1-1ubuntu0.22.04.2 [181 kB] 2024-07-15T06:34:06.264 INFO:teuthology.orchestra.run.smithi134.stdout:Get:73 http://archive.ubuntu.com/ubuntu jammy/main amd64 python-babel-localedata all 2.8.0+dfsg.1-7 [4,982 kB] 2024-07-15T06:34:06.462 INFO:teuthology.orchestra.run.smithi134.stdout:Get:74 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 rbd-fuse amd64 17.2.7-1jammy [75.8 kB] 2024-07-15T06:34:06.462 INFO:teuthology.orchestra.run.smithi134.stdout:Get:75 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-babel all 2.8.0+dfsg.1-7 [85.1 kB] 2024-07-15T06:34:06.464 INFO:teuthology.orchestra.run.smithi134.stdout:Get:76 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-jinja2 all 3.0.3-1ubuntu0.2 [108 kB] 2024-07-15T06:34:06.466 INFO:teuthology.orchestra.run.smithi134.stdout:Get:77 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-repoze.lru all 0.7-2 [12.1 kB] 2024-07-15T06:34:06.466 INFO:teuthology.orchestra.run.smithi134.stdout:Get:78 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-routes all 2.5.1-1ubuntu1 [89.0 kB] 2024-07-15T06:34:06.469 INFO:teuthology.orchestra.run.smithi134.stdout:Get:79 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-sklearn-lib amd64 0.23.2-5ubuntu6 [2,058 kB] 2024-07-15T06:34:06.508 INFO:teuthology.orchestra.run.smithi134.stdout:Get:80 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-joblib all 0.17.0-4ubuntu1 [204 kB] 2024-07-15T06:34:06.511 INFO:teuthology.orchestra.run.smithi134.stdout:Get:81 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-threadpoolctl all 3.1.0-1 [21.3 kB] 2024-07-15T06:34:06.512 INFO:teuthology.orchestra.run.smithi134.stdout:Get:82 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-sklearn all 0.23.2-5ubuntu6 [1,829 kB] 2024-07-15T06:34:06.560 INFO:teuthology.orchestra.run.smithi134.stdout:Get:83 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-cachetools all 5.0.0-1 [9,722 B] 2024-07-15T06:34:06.560 INFO:teuthology.orchestra.run.smithi134.stdout:Get:84 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pyasn1 all 0.4.8-1 [50.9 kB] 2024-07-15T06:34:06.628 INFO:teuthology.orchestra.run.smithi134.stdout:Get:85 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pyasn1-modules all 0.2.1-1 [33.0 kB] 2024-07-15T06:34:06.629 INFO:teuthology.orchestra.run.smithi134.stdout:Get:86 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-rsa all 4.8-1 [28.4 kB] 2024-07-15T06:34:06.630 INFO:teuthology.orchestra.run.smithi134.stdout:Get:87 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-google-auth all 1.5.1-3 [35.7 kB] 2024-07-15T06:34:06.631 INFO:teuthology.orchestra.run.smithi134.stdout:Get:88 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-requests-oauthlib all 1.3.0+ds-0.1 [18.7 kB] 2024-07-15T06:34:06.632 INFO:teuthology.orchestra.run.smithi134.stdout:Get:89 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-websocket all 1.2.3-1 [34.7 kB] 2024-07-15T06:34:06.633 INFO:teuthology.orchestra.run.smithi134.stdout:Get:90 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-kubernetes all 12.0.1-1ubuntu1 [353 kB] 2024-07-15T06:34:06.640 INFO:teuthology.orchestra.run.smithi134.stdout:Get:91 http://archive.ubuntu.com/ubuntu jammy/main amd64 libonig5 amd64 6.9.7.1-2build1 [172 kB] 2024-07-15T06:34:06.710 INFO:teuthology.orchestra.run.smithi134.stdout:Get:92 http://archive.ubuntu.com/ubuntu jammy/main amd64 libjq1 amd64 1.6-2.1ubuntu3 [133 kB] 2024-07-15T06:34:06.713 INFO:teuthology.orchestra.run.smithi134.stdout:Get:93 http://archive.ubuntu.com/ubuntu jammy/main amd64 jq amd64 1.6-2.1ubuntu3 [52.5 kB] 2024-07-15T06:34:06.715 INFO:teuthology.orchestra.run.smithi134.stdout:Get:94 http://archive.ubuntu.com/ubuntu jammy/main amd64 socat amd64 1.7.4.1-3ubuntu4 [349 kB] 2024-07-15T06:34:06.791 INFO:teuthology.orchestra.run.smithi134.stdout:Get:95 http://archive.ubuntu.com/ubuntu jammy/universe amd64 xmlstarlet amd64 1.6.1-2.1 [265 kB] 2024-07-15T06:34:06.797 INFO:teuthology.orchestra.run.smithi134.stdout:Get:96 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 nvme-cli amd64 1.16-3ubuntu0.1 [474 kB] 2024-07-15T06:34:06.808 INFO:teuthology.orchestra.run.smithi134.stdout:Get:97 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-iniconfig all 1.1.1-2 [6,024 B] 2024-07-15T06:34:06.809 INFO:teuthology.orchestra.run.smithi134.stdout:Get:98 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pastescript all 2.0.2-4 [54.6 kB] 2024-07-15T06:34:06.810 INFO:teuthology.orchestra.run.smithi134.stdout:Get:99 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-pluggy all 0.13.0-7.1 [19.0 kB] 2024-07-15T06:34:06.811 INFO:teuthology.orchestra.run.smithi134.stdout:Get:100 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-psutil amd64 5.9.0-1build1 [158 kB] 2024-07-15T06:34:06.815 INFO:teuthology.orchestra.run.smithi134.stdout:Get:101 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-py all 1.10.0-1 [71.9 kB] 2024-07-15T06:34:06.872 INFO:teuthology.orchestra.run.smithi134.stdout:Get:102 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pygments all 2.11.2+dfsg-2 [750 kB] 2024-07-15T06:34:06.889 INFO:teuthology.orchestra.run.smithi134.stdout:Get:103 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pyinotify all 0.9.6-1.3 [24.8 kB] 2024-07-15T06:34:06.890 INFO:teuthology.orchestra.run.smithi134.stdout:Get:104 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-toml all 0.10.2-1 [16.5 kB] 2024-07-15T06:34:06.954 INFO:teuthology.orchestra.run.smithi134.stdout:Get:105 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-pytest all 6.2.5-1ubuntu2 [214 kB] 2024-07-15T06:34:06.959 INFO:teuthology.orchestra.run.smithi134.stdout:Get:106 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-simplejson amd64 3.17.6-1build1 [54.7 kB] 2024-07-15T06:34:06.961 INFO:teuthology.orchestra.run.smithi134.stdout:Get:107 http://archive.ubuntu.com/ubuntu jammy/universe amd64 qttranslations5-l10n all 5.15.3-1 [1,983 kB] 2024-07-15T06:34:07.564 INFO:teuthology.orchestra.run.smithi134.stdout:Fetched 187 MB in 2s (82.1 MB/s) 2024-07-15T06:34:07.689 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package libfmt8:amd64. 2024-07-15T06:34:07.788 INFO:teuthology.orchestra.run.smithi134.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 165982 files and directories currently installed.) 2024-07-15T06:34:07.820 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../000-libfmt8_8.1.1+ds1-2_amd64.deb ... 2024-07-15T06:34:07.879 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking libfmt8:amd64 (8.1.1+ds1-2) ... 2024-07-15T06:34:08.232 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package liblttng-ust1:amd64. 2024-07-15T06:34:08.258 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../001-liblttng-ust1_2.13.1-1ubuntu1_amd64.deb ... 2024-07-15T06:34:08.299 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking liblttng-ust1:amd64 (2.13.1-1ubuntu1) ... 2024-07-15T06:34:08.629 INFO:teuthology.orchestra.run.smithi089.stdout:Hit:7 http://archive.ubuntu.com/ubuntu jammy InRelease 2024-07-15T06:34:08.646 INFO:teuthology.orchestra.run.smithi089.stdout:Hit:8 http://archive.ubuntu.com/ubuntu jammy-updates InRelease 2024-07-15T06:34:08.664 INFO:teuthology.orchestra.run.smithi089.stdout:Hit:9 http://archive.ubuntu.com/ubuntu jammy-backports InRelease 2024-07-15T06:34:08.735 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package librados2. 2024-07-15T06:34:08.762 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../002-librados2_17.2.7-1jammy_amd64.deb ... 2024-07-15T06:34:08.802 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking librados2 (17.2.7-1jammy) ... 2024-07-15T06:34:08.958 INFO:teuthology.orchestra.run.smithi089.stdout:Fetched 28.3 kB in 6s (4,606 B/s) 2024-07-15T06:34:09.255 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package librbd1. 2024-07-15T06:34:09.281 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../003-librbd1_17.2.7-1jammy_amd64.deb ... 2024-07-15T06:34:09.313 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking librbd1 (17.2.7-1jammy) ... 2024-07-15T06:34:09.782 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package libcephfs2. 2024-07-15T06:34:09.809 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../004-libcephfs2_17.2.7-1jammy_amd64.deb ... 2024-07-15T06:34:09.849 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking libcephfs2 (17.2.7-1jammy) ... 2024-07-15T06:34:09.932 INFO:teuthology.orchestra.run.smithi089.stdout:Reading package lists... 2024-07-15T06:34:09.948 DEBUG:teuthology.orchestra.run.smithi089:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install ceph=17.2.7-1jammy cephadm=17.2.7-1jammy ceph-mds=17.2.7-1jammy ceph-mgr=17.2.7-1jammy ceph-common=17.2.7-1jammy ceph-fuse=17.2.7-1jammy ceph-test=17.2.7-1jammy radosgw=17.2.7-1jammy python3-rados=17.2.7-1jammy python3-rgw=17.2.7-1jammy python3-cephfs=17.2.7-1jammy python3-rbd=17.2.7-1jammy libcephfs2=17.2.7-1jammy libcephfs-dev=17.2.7-1jammy librados2=17.2.7-1jammy librbd1=17.2.7-1jammy rbd-fuse=17.2.7-1jammy 2024-07-15T06:34:10.008 INFO:teuthology.orchestra.run.smithi089.stdout:Reading package lists... 2024-07-15T06:34:10.126 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package python3-rados. 2024-07-15T06:34:10.153 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../005-python3-rados_17.2.7-1jammy_amd64.deb ... 2024-07-15T06:34:10.193 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking python3-rados (17.2.7-1jammy) ... 2024-07-15T06:34:10.211 INFO:teuthology.orchestra.run.smithi089.stdout:Building dependency tree... 2024-07-15T06:34:10.212 INFO:teuthology.orchestra.run.smithi089.stdout:Reading state information... 2024-07-15T06:34:10.399 INFO:teuthology.orchestra.run.smithi089.stdout:The following packages were automatically installed and are no longer required: 2024-07-15T06:34:10.399 INFO:teuthology.orchestra.run.smithi089.stdout: libboost-iostreams1.74.0 libboost-thread1.74.0 libgfapi0 libgfrpc0 libgfxdr0 2024-07-15T06:34:10.399 INFO:teuthology.orchestra.run.smithi089.stdout: libglusterfs0 libiscsi7 libpmemobj1 libpython2-dev libpython2-stdlib 2024-07-15T06:34:10.400 INFO:teuthology.orchestra.run.smithi089.stdout: libpython2.7 libpython2.7-dev libpython2.7-minimal libpython2.7-stdlib 2024-07-15T06:34:10.400 INFO:teuthology.orchestra.run.smithi089.stdout: linux-image-5.4.0-135-generic linux-modules-5.4.0-135-generic python2 2024-07-15T06:34:10.400 INFO:teuthology.orchestra.run.smithi089.stdout: python2-dev python2-minimal python2.7 python2.7-dev python2.7-minimal 2024-07-15T06:34:10.400 INFO:teuthology.orchestra.run.smithi089.stdout:Use 'sudo apt autoremove' to remove them. 2024-07-15T06:34:10.401 INFO:teuthology.orchestra.run.smithi089.stdout:The following additional packages will be installed: 2024-07-15T06:34:10.401 INFO:teuthology.orchestra.run.smithi089.stdout: ceph-base ceph-mgr-cephadm ceph-mgr-dashboard ceph-mgr-diskprediction-local 2024-07-15T06:34:10.401 INFO:teuthology.orchestra.run.smithi089.stdout: ceph-mgr-k8sevents ceph-mgr-modules-core ceph-mon ceph-osd ceph-volume jq 2024-07-15T06:34:10.401 INFO:teuthology.orchestra.run.smithi089.stdout: libdouble-conversion3 libfmt8 libjq1 liblttng-ust1 liboath0 libonig5 2024-07-15T06:34:10.401 INFO:teuthology.orchestra.run.smithi089.stdout: libpcre2-16-0 libqt5core5a libqt5dbus5 libqt5network5 libradosstriper1 2024-07-15T06:34:10.402 INFO:teuthology.orchestra.run.smithi089.stdout: librdkafka1 librgw2 libsqlite3-mod-ceph libthrift-0.16.0 nvme-cli 2024-07-15T06:34:10.402 INFO:teuthology.orchestra.run.smithi089.stdout: python-babel-localedata python-pastedeploy-tpl python3-babel python3-bcrypt 2024-07-15T06:34:10.402 INFO:teuthology.orchestra.run.smithi089.stdout: python3-cachetools python3-ceph-argparse python3-ceph-common python3-certifi 2024-07-15T06:34:10.402 INFO:teuthology.orchestra.run.smithi089.stdout: python3-cheroot python3-cherrypy3 python3-google-auth python3-idna 2024-07-15T06:34:10.402 INFO:teuthology.orchestra.run.smithi089.stdout: python3-iniconfig python3-jaraco.classes python3-jaraco.collections 2024-07-15T06:34:10.402 INFO:teuthology.orchestra.run.smithi089.stdout: python3-jaraco.functools python3-jaraco.text python3-jinja2 python3-joblib 2024-07-15T06:34:10.402 INFO:teuthology.orchestra.run.smithi089.stdout: python3-kubernetes python3-logutils python3-mako python3-markupsafe 2024-07-15T06:34:10.402 INFO:teuthology.orchestra.run.smithi089.stdout: python3-natsort python3-openssl python3-paste python3-pastedeploy 2024-07-15T06:34:10.403 INFO:teuthology.orchestra.run.smithi089.stdout: python3-pastescript python3-pecan python3-pluggy python3-portend 2024-07-15T06:34:10.403 INFO:teuthology.orchestra.run.smithi089.stdout: python3-prettytable python3-psutil python3-py python3-pyasn1 2024-07-15T06:34:10.403 INFO:teuthology.orchestra.run.smithi089.stdout: python3-pyasn1-modules python3-pygments python3-pyinotify python3-pytest 2024-07-15T06:34:10.403 INFO:teuthology.orchestra.run.smithi089.stdout: python3-repoze.lru python3-requests python3-requests-oauthlib python3-routes 2024-07-15T06:34:10.403 INFO:teuthology.orchestra.run.smithi089.stdout: python3-rsa python3-simplegeneric python3-simplejson python3-singledispatch 2024-07-15T06:34:10.403 INFO:teuthology.orchestra.run.smithi089.stdout: python3-sklearn python3-sklearn-lib python3-tempita python3-tempora 2024-07-15T06:34:10.403 INFO:teuthology.orchestra.run.smithi089.stdout: python3-threadpoolctl python3-toml python3-urllib3 python3-waitress 2024-07-15T06:34:10.403 INFO:teuthology.orchestra.run.smithi089.stdout: python3-wcwidth python3-webob python3-websocket python3-webtest 2024-07-15T06:34:10.403 INFO:teuthology.orchestra.run.smithi089.stdout: python3-werkzeug python3-zc.lockfile qttranslations5-l10n socat xmlstarlet 2024-07-15T06:34:10.403 INFO:teuthology.orchestra.run.smithi089.stdout:Suggested packages: 2024-07-15T06:34:10.404 INFO:teuthology.orchestra.run.smithi089.stdout: python3-influxdb python-jinja2-doc python3-beaker python-mako-doc 2024-07-15T06:34:10.404 INFO:teuthology.orchestra.run.smithi089.stdout: python-natsort-doc python-openssl-doc python3-openssl-dbg httpd-wsgi 2024-07-15T06:34:10.404 INFO:teuthology.orchestra.run.smithi089.stdout: libapache2-mod-python libapache2-mod-scgi libjs-mochikit python-pecan-doc 2024-07-15T06:34:10.404 INFO:teuthology.orchestra.run.smithi089.stdout: python-psutil-doc subversion python-pygments-doc python-pyinotify-doc 2024-07-15T06:34:10.404 INFO:teuthology.orchestra.run.smithi089.stdout: python3-socks python-requests-doc python3-dap python-sklearn-doc ipython3 2024-07-15T06:34:10.404 INFO:teuthology.orchestra.run.smithi089.stdout: python-waitress-doc python-webob-doc python-webtest-doc python-werkzeug-doc 2024-07-15T06:34:10.404 INFO:teuthology.orchestra.run.smithi089.stdout: python3-watchdog 2024-07-15T06:34:10.404 INFO:teuthology.orchestra.run.smithi089.stdout:Recommended packages: 2024-07-15T06:34:10.404 INFO:teuthology.orchestra.run.smithi089.stdout: btrfs-tools 2024-07-15T06:34:10.487 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package python3-ceph-argparse. 2024-07-15T06:34:10.501 INFO:teuthology.orchestra.run.smithi089.stdout:The following NEW packages will be installed: 2024-07-15T06:34:10.502 INFO:teuthology.orchestra.run.smithi089.stdout: ceph ceph-base ceph-common ceph-fuse ceph-mds ceph-mgr ceph-mgr-cephadm 2024-07-15T06:34:10.502 INFO:teuthology.orchestra.run.smithi089.stdout: ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-k8sevents 2024-07-15T06:34:10.502 INFO:teuthology.orchestra.run.smithi089.stdout: ceph-mgr-modules-core ceph-mon ceph-osd ceph-test ceph-volume cephadm jq 2024-07-15T06:34:10.503 INFO:teuthology.orchestra.run.smithi089.stdout: libcephfs-dev libcephfs2 libdouble-conversion3 libfmt8 libjq1 liblttng-ust1 2024-07-15T06:34:10.503 INFO:teuthology.orchestra.run.smithi089.stdout: liboath0 libonig5 libpcre2-16-0 libqt5core5a libqt5dbus5 libqt5network5 2024-07-15T06:34:10.503 INFO:teuthology.orchestra.run.smithi089.stdout: librados2 libradosstriper1 librbd1 librdkafka1 librgw2 libsqlite3-mod-ceph 2024-07-15T06:34:10.503 INFO:teuthology.orchestra.run.smithi089.stdout: libthrift-0.16.0 nvme-cli python-babel-localedata python-pastedeploy-tpl 2024-07-15T06:34:10.503 INFO:teuthology.orchestra.run.smithi089.stdout: python3-babel python3-bcrypt python3-cachetools python3-ceph-argparse 2024-07-15T06:34:10.503 INFO:teuthology.orchestra.run.smithi089.stdout: python3-ceph-common python3-cephfs python3-certifi python3-cheroot 2024-07-15T06:34:10.503 INFO:teuthology.orchestra.run.smithi089.stdout: python3-cherrypy3 python3-google-auth python3-idna python3-iniconfig 2024-07-15T06:34:10.503 INFO:teuthology.orchestra.run.smithi089.stdout: python3-jaraco.classes python3-jaraco.collections python3-jaraco.functools 2024-07-15T06:34:10.503 INFO:teuthology.orchestra.run.smithi089.stdout: python3-jaraco.text python3-jinja2 python3-joblib python3-kubernetes 2024-07-15T06:34:10.504 INFO:teuthology.orchestra.run.smithi089.stdout: python3-logutils python3-mako python3-markupsafe python3-natsort 2024-07-15T06:34:10.504 INFO:teuthology.orchestra.run.smithi089.stdout: python3-openssl python3-paste python3-pastedeploy python3-pastescript 2024-07-15T06:34:10.504 INFO:teuthology.orchestra.run.smithi089.stdout: python3-pecan python3-pluggy python3-portend python3-prettytable 2024-07-15T06:34:10.504 INFO:teuthology.orchestra.run.smithi089.stdout: python3-psutil python3-py python3-pyasn1 python3-pyasn1-modules 2024-07-15T06:34:10.504 INFO:teuthology.orchestra.run.smithi089.stdout: python3-pygments python3-pyinotify python3-pytest python3-rados python3-rbd 2024-07-15T06:34:10.504 INFO:teuthology.orchestra.run.smithi089.stdout: python3-repoze.lru python3-requests python3-requests-oauthlib python3-rgw 2024-07-15T06:34:10.504 INFO:teuthology.orchestra.run.smithi089.stdout: python3-routes python3-rsa python3-simplegeneric python3-simplejson 2024-07-15T06:34:10.504 INFO:teuthology.orchestra.run.smithi089.stdout: python3-singledispatch python3-sklearn python3-sklearn-lib python3-tempita 2024-07-15T06:34:10.504 INFO:teuthology.orchestra.run.smithi089.stdout: python3-tempora python3-threadpoolctl python3-toml python3-urllib3 2024-07-15T06:34:10.504 INFO:teuthology.orchestra.run.smithi089.stdout: python3-waitress python3-wcwidth python3-webob python3-websocket 2024-07-15T06:34:10.504 INFO:teuthology.orchestra.run.smithi089.stdout: python3-webtest python3-werkzeug python3-zc.lockfile qttranslations5-l10n 2024-07-15T06:34:10.504 INFO:teuthology.orchestra.run.smithi089.stdout: radosgw rbd-fuse socat xmlstarlet 2024-07-15T06:34:10.513 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../006-python3-ceph-argparse_17.2.7-1jammy_all.deb ... 2024-07-15T06:34:10.541 INFO:teuthology.orchestra.run.smithi089.stdout:0 upgraded, 107 newly installed, 0 to remove and 316 not upgraded. 2024-07-15T06:34:10.541 INFO:teuthology.orchestra.run.smithi089.stdout:Need to get 187 MB of archives. 2024-07-15T06:34:10.541 INFO:teuthology.orchestra.run.smithi089.stdout:After this operation, 815 MB of additional disk space will be used. 2024-07-15T06:34:10.541 INFO:teuthology.orchestra.run.smithi089.stdout:Get:1 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 librados2 amd64 17.2.7-1jammy [3,198 kB] 2024-07-15T06:34:10.553 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking python3-ceph-argparse (17.2.7-1jammy) ... 2024-07-15T06:34:10.569 INFO:teuthology.orchestra.run.smithi089.stdout:Get:2 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 librbd1 amd64 17.2.7-1jammy [3,067 kB] 2024-07-15T06:34:10.593 INFO:teuthology.orchestra.run.smithi089.stdout:Get:3 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 libcephfs2 amd64 17.2.7-1jammy [632 kB] 2024-07-15T06:34:10.597 INFO:teuthology.orchestra.run.smithi089.stdout:Get:4 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 python3-rados amd64 17.2.7-1jammy [344 kB] 2024-07-15T06:34:10.600 INFO:teuthology.orchestra.run.smithi089.stdout:Get:5 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 python3-ceph-argparse all 17.2.7-1jammy [23.1 kB] 2024-07-15T06:34:10.600 INFO:teuthology.orchestra.run.smithi089.stdout:Get:6 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 python3-cephfs amd64 17.2.7-1jammy [164 kB] 2024-07-15T06:34:10.602 INFO:teuthology.orchestra.run.smithi089.stdout:Get:7 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 python3-ceph-common all 17.2.7-1jammy [47.7 kB] 2024-07-15T06:34:10.602 INFO:teuthology.orchestra.run.smithi089.stdout:Get:8 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 python3-rbd amd64 17.2.7-1jammy [313 kB] 2024-07-15T06:34:10.605 INFO:teuthology.orchestra.run.smithi089.stdout:Get:9 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 librgw2 amd64 17.2.7-1jammy [5,086 kB] 2024-07-15T06:34:10.640 INFO:teuthology.orchestra.run.smithi089.stdout:Get:10 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 python3-rgw amd64 17.2.7-1jammy [102 kB] 2024-07-15T06:34:10.640 INFO:teuthology.orchestra.run.smithi089.stdout:Get:11 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 libradosstriper1 amd64 17.2.7-1jammy [415 kB] 2024-07-15T06:34:10.644 INFO:teuthology.orchestra.run.smithi089.stdout:Get:12 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-common amd64 17.2.7-1jammy [25.3 MB] 2024-07-15T06:34:10.689 INFO:teuthology.orchestra.run.smithi089.stdout:Get:13 http://archive.ubuntu.com/ubuntu jammy/universe amd64 libfmt8 amd64 8.1.1+ds1-2 [60.2 kB] 2024-07-15T06:34:10.806 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package python3-cephfs. 2024-07-15T06:34:10.821 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../007-python3-cephfs_17.2.7-1jammy_amd64.deb ... 2024-07-15T06:34:10.874 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking python3-cephfs (17.2.7-1jammy) ... 2024-07-15T06:34:10.958 INFO:teuthology.orchestra.run.smithi089.stdout:Get:14 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-base amd64 17.2.7-1jammy [6,378 kB] 2024-07-15T06:34:10.967 INFO:teuthology.orchestra.run.smithi089.stdout:Get:15 http://archive.ubuntu.com/ubuntu jammy/main amd64 liblttng-ust1 amd64 2.13.1-1ubuntu1 [190 kB] 2024-07-15T06:34:11.005 INFO:teuthology.orchestra.run.smithi089.stdout:Get:16 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-modules-core all 17.2.7-1jammy [224 kB] 2024-07-15T06:34:11.007 INFO:teuthology.orchestra.run.smithi089.stdout:Get:17 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 libsqlite3-mod-ceph amd64 17.2.7-1jammy [115 kB] 2024-07-15T06:34:11.007 INFO:teuthology.orchestra.run.smithi089.stdout:Get:18 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr amd64 17.2.7-1jammy [1,429 kB] 2024-07-15T06:34:11.018 INFO:teuthology.orchestra.run.smithi089.stdout:Get:19 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mon amd64 17.2.7-1jammy [8,201 kB] 2024-07-15T06:34:11.075 INFO:teuthology.orchestra.run.smithi089.stdout:Get:20 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-osd amd64 17.2.7-1jammy [28.4 MB] 2024-07-15T06:34:11.138 INFO:teuthology.orchestra.run.smithi089.stdout:Get:21 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-wcwidth all 0.2.5+dfsg1-1 [21.9 kB] 2024-07-15T06:34:11.146 INFO:teuthology.orchestra.run.smithi089.stdout:Get:22 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-prettytable all 2.5.0-2 [31.3 kB] 2024-07-15T06:34:11.150 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package python3-ceph-common. 2024-07-15T06:34:11.158 INFO:teuthology.orchestra.run.smithi089.stdout:Get:23 http://archive.ubuntu.com/ubuntu jammy/universe amd64 librdkafka1 amd64 1.8.0-1build1 [633 kB] 2024-07-15T06:34:11.166 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../008-python3-ceph-common_17.2.7-1jammy_all.deb ... 2024-07-15T06:34:11.200 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking python3-ceph-common (17.2.7-1jammy) ... 2024-07-15T06:34:11.345 INFO:teuthology.orchestra.run.smithi089.stdout:Get:24 http://archive.ubuntu.com/ubuntu jammy/universe amd64 libdouble-conversion3 amd64 3.1.7-4 [39.0 kB] 2024-07-15T06:34:11.420 INFO:teuthology.orchestra.run.smithi089.stdout:Get:25 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph amd64 17.2.7-1jammy [3,690 B] 2024-07-15T06:34:11.420 INFO:teuthology.orchestra.run.smithi089.stdout:Get:26 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libpcre2-16-0 amd64 10.39-3ubuntu0.1 [203 kB] 2024-07-15T06:34:11.421 INFO:teuthology.orchestra.run.smithi089.stdout:Get:27 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-fuse amd64 17.2.7-1jammy [783 kB] 2024-07-15T06:34:11.425 INFO:teuthology.orchestra.run.smithi089.stdout:Get:28 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5core5a amd64 5.15.3+dfsg-2ubuntu0.2 [2,006 kB] 2024-07-15T06:34:11.426 INFO:teuthology.orchestra.run.smithi089.stdout:Get:29 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mds amd64 17.2.7-1jammy [2,120 kB] 2024-07-15T06:34:11.441 INFO:teuthology.orchestra.run.smithi089.stdout:Get:30 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 cephadm amd64 17.2.7-1jammy [80.2 kB] 2024-07-15T06:34:11.442 INFO:teuthology.orchestra.run.smithi089.stdout:Get:31 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-cephadm all 17.2.7-1jammy [118 kB] 2024-07-15T06:34:11.443 INFO:teuthology.orchestra.run.smithi089.stdout:Get:32 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-dashboard all 17.2.7-1jammy [1,824 kB] 2024-07-15T06:34:11.456 INFO:teuthology.orchestra.run.smithi089.stdout:Get:33 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-diskprediction-local all 17.2.7-1jammy [8,615 kB] 2024-07-15T06:34:11.470 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package python3-wcwidth. 2024-07-15T06:34:11.475 INFO:teuthology.orchestra.run.smithi089.stdout:Get:34 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5dbus5 amd64 5.15.3+dfsg-2ubuntu0.2 [222 kB] 2024-07-15T06:34:11.481 INFO:teuthology.orchestra.run.smithi089.stdout:Get:35 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5network5 amd64 5.15.3+dfsg-2ubuntu0.2 [731 kB] 2024-07-15T06:34:11.496 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../009-python3-wcwidth_0.2.5+dfsg1-1_all.deb ... 2024-07-15T06:34:11.504 INFO:teuthology.orchestra.run.smithi089.stdout:Get:36 http://archive.ubuntu.com/ubuntu jammy/universe amd64 libthrift-0.16.0 amd64 0.16.0-2 [267 kB] 2024-07-15T06:34:11.510 INFO:teuthology.orchestra.run.smithi089.stdout:Get:37 http://archive.ubuntu.com/ubuntu jammy/main amd64 liboath0 amd64 2.6.7-3build1 [41.1 kB] 2024-07-15T06:34:11.512 INFO:teuthology.orchestra.run.smithi089.stdout:Get:38 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-openssl all 21.0.0-1 [45.2 kB] 2024-07-15T06:34:11.513 INFO:teuthology.orchestra.run.smithi089.stdout:Get:39 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-bcrypt amd64 3.2.0-1build1 [32.7 kB] 2024-07-15T06:34:11.514 INFO:teuthology.orchestra.run.smithi089.stdout:Get:40 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.functools all 3.4.0-2 [9,030 B] 2024-07-15T06:34:11.515 INFO:teuthology.orchestra.run.smithi089.stdout:Get:41 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-k8sevents all 17.2.7-1jammy [3,814 B] 2024-07-15T06:34:11.515 INFO:teuthology.orchestra.run.smithi089.stdout:Get:42 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-test amd64 17.2.7-1jammy [52.7 MB] 2024-07-15T06:34:11.528 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking python3-wcwidth (0.2.5+dfsg1-1) ... 2024-07-15T06:34:11.585 INFO:teuthology.orchestra.run.smithi089.stdout:Get:43 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-cheroot all 8.5.2+ds1-1ubuntu3.1 [71.1 kB] 2024-07-15T06:34:11.587 INFO:teuthology.orchestra.run.smithi089.stdout:Get:44 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.classes all 3.2.1-3 [6,452 B] 2024-07-15T06:34:11.587 INFO:teuthology.orchestra.run.smithi089.stdout:Get:45 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.text all 3.6.0-2 [8,716 B] 2024-07-15T06:34:11.588 INFO:teuthology.orchestra.run.smithi089.stdout:Get:46 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.collections all 3.4.0-2 [11.4 kB] 2024-07-15T06:34:11.667 INFO:teuthology.orchestra.run.smithi089.stdout:Get:47 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-tempora all 4.1.2-1 [14.8 kB] 2024-07-15T06:34:11.667 INFO:teuthology.orchestra.run.smithi089.stdout:Get:48 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-portend all 3.0.0-1 [7,240 B] 2024-07-15T06:34:11.668 INFO:teuthology.orchestra.run.smithi089.stdout:Get:49 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-zc.lockfile all 2.0-1 [8,980 B] 2024-07-15T06:34:11.668 INFO:teuthology.orchestra.run.smithi089.stdout:Get:50 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-cherrypy3 all 18.6.1-4 [208 kB] 2024-07-15T06:34:11.674 INFO:teuthology.orchestra.run.smithi089.stdout:Get:51 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-natsort all 8.0.2-1 [35.3 kB] 2024-07-15T06:34:11.675 INFO:teuthology.orchestra.run.smithi089.stdout:Get:52 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-logutils all 0.3.3-8 [17.6 kB] 2024-07-15T06:34:11.748 INFO:teuthology.orchestra.run.smithi089.stdout:Get:53 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-markupsafe amd64 2.0.1-2build1 [12.7 kB] 2024-07-15T06:34:11.748 INFO:teuthology.orchestra.run.smithi089.stdout:Get:54 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-mako all 1.1.3+ds1-2ubuntu0.1 [60.5 kB] 2024-07-15T06:34:11.750 INFO:teuthology.orchestra.run.smithi089.stdout:Get:55 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-simplegeneric all 0.8.1-3 [11.3 kB] 2024-07-15T06:34:11.751 INFO:teuthology.orchestra.run.smithi089.stdout:Get:56 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-singledispatch all 3.4.0.3-3 [7,320 B] 2024-07-15T06:34:11.830 INFO:teuthology.orchestra.run.smithi089.stdout:Get:57 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-webob all 1:1.8.6-1.1 [85.2 kB] 2024-07-15T06:34:11.832 INFO:teuthology.orchestra.run.smithi089.stdout:Get:58 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-waitress all 1.4.4-1.1ubuntu1 [47.3 kB] 2024-07-15T06:34:11.833 INFO:teuthology.orchestra.run.smithi089.stdout:Get:59 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-tempita all 0.5.2-6ubuntu1 [15.1 kB] 2024-07-15T06:34:11.834 INFO:teuthology.orchestra.run.smithi089.stdout:Get:60 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-paste all 3.5.0+dfsg1-1 [456 kB] 2024-07-15T06:34:11.844 INFO:teuthology.orchestra.run.smithi089.stdout:Get:61 http://archive.ubuntu.com/ubuntu jammy/main amd64 python-pastedeploy-tpl all 2.1.1-1 [4,892 B] 2024-07-15T06:34:11.845 INFO:teuthology.orchestra.run.smithi089.stdout:Get:62 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pastedeploy all 2.1.1-1 [26.6 kB] 2024-07-15T06:34:11.856 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package python3-prettytable. 2024-07-15T06:34:11.882 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../010-python3-prettytable_2.5.0-2_all.deb ... 2024-07-15T06:34:11.911 INFO:teuthology.orchestra.run.smithi089.stdout:Get:63 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-webtest all 2.0.35-1 [28.5 kB] 2024-07-15T06:34:11.922 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking python3-prettytable (2.5.0-2) ... 2024-07-15T06:34:11.937 INFO:teuthology.orchestra.run.smithi089.stdout:Get:64 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pecan all 1.3.3-4ubuntu2 [87.3 kB] 2024-07-15T06:34:12.018 INFO:teuthology.orchestra.run.smithi089.stdout:Get:65 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-certifi all 2020.6.20-1 [150 kB] 2024-07-15T06:34:12.117 INFO:teuthology.orchestra.run.smithi089.stdout:Get:66 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-idna all 3.3-1ubuntu0.1 [52.1 kB] 2024-07-15T06:34:12.118 INFO:teuthology.orchestra.run.smithi089.stdout:Get:67 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-volume all 17.2.7-1jammy [240 kB] 2024-07-15T06:34:12.118 INFO:teuthology.orchestra.run.smithi089.stdout:Get:68 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-urllib3 all 1.26.5-1~exp1ubuntu0.1 [98.2 kB] 2024-07-15T06:34:12.120 INFO:teuthology.orchestra.run.smithi089.stdout:Get:69 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 libcephfs-dev amd64 17.2.7-1jammy [22.4 kB] 2024-07-15T06:34:12.185 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package python3-rbd. 2024-07-15T06:34:12.201 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../011-python3-rbd_17.2.7-1jammy_amd64.deb ... 2024-07-15T06:34:12.203 INFO:teuthology.orchestra.run.smithi089.stdout:Get:70 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 radosgw amd64 17.2.7-1jammy [15.4 MB] 2024-07-15T06:34:12.203 INFO:teuthology.orchestra.run.smithi089.stdout:Get:71 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-requests all 2.25.1+dfsg-2ubuntu0.1 [48.8 kB] 2024-07-15T06:34:12.205 INFO:teuthology.orchestra.run.smithi089.stdout:Get:72 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-werkzeug all 2.0.2+dfsg1-1ubuntu0.22.04.2 [181 kB] 2024-07-15T06:34:12.209 INFO:teuthology.orchestra.run.smithi089.stdout:Get:73 http://archive.ubuntu.com/ubuntu jammy/main amd64 python-babel-localedata all 2.8.0+dfsg.1-7 [4,982 kB] 2024-07-15T06:34:12.243 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking python3-rbd (17.2.7-1jammy) ... 2024-07-15T06:34:12.427 INFO:teuthology.orchestra.run.smithi089.stdout:Get:74 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 rbd-fuse amd64 17.2.7-1jammy [75.8 kB] 2024-07-15T06:34:12.427 INFO:teuthology.orchestra.run.smithi089.stdout:Get:75 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-babel all 2.8.0+dfsg.1-7 [85.1 kB] 2024-07-15T06:34:12.429 INFO:teuthology.orchestra.run.smithi089.stdout:Get:76 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-jinja2 all 3.0.3-1ubuntu0.2 [108 kB] 2024-07-15T06:34:12.432 INFO:teuthology.orchestra.run.smithi089.stdout:Get:77 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-repoze.lru all 0.7-2 [12.1 kB] 2024-07-15T06:34:12.432 INFO:teuthology.orchestra.run.smithi089.stdout:Get:78 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-routes all 2.5.1-1ubuntu1 [89.0 kB] 2024-07-15T06:34:12.435 INFO:teuthology.orchestra.run.smithi089.stdout:Get:79 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-sklearn-lib amd64 0.23.2-5ubuntu6 [2,058 kB] 2024-07-15T06:34:12.478 INFO:teuthology.orchestra.run.smithi089.stdout:Get:80 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-joblib all 0.17.0-4ubuntu1 [204 kB] 2024-07-15T06:34:12.482 INFO:teuthology.orchestra.run.smithi089.stdout:Get:81 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-threadpoolctl all 3.1.0-1 [21.3 kB] 2024-07-15T06:34:12.483 INFO:teuthology.orchestra.run.smithi089.stdout:Get:82 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-sklearn all 0.23.2-5ubuntu6 [1,829 kB] 2024-07-15T06:34:12.542 INFO:teuthology.orchestra.run.smithi089.stdout:Get:83 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-cachetools all 5.0.0-1 [9,722 B] 2024-07-15T06:34:12.542 INFO:teuthology.orchestra.run.smithi089.stdout:Get:84 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pyasn1 all 0.4.8-1 [50.9 kB] 2024-07-15T06:34:12.572 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package librdkafka1:amd64. 2024-07-15T06:34:12.594 INFO:teuthology.orchestra.run.smithi089.stdout:Get:85 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pyasn1-modules all 0.2.1-1 [33.0 kB] 2024-07-15T06:34:12.595 INFO:teuthology.orchestra.run.smithi089.stdout:Get:86 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-rsa all 4.8-1 [28.4 kB] 2024-07-15T06:34:12.596 INFO:teuthology.orchestra.run.smithi089.stdout:Get:87 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-google-auth all 1.5.1-3 [35.7 kB] 2024-07-15T06:34:12.598 INFO:teuthology.orchestra.run.smithi089.stdout:Get:88 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-requests-oauthlib all 1.3.0+ds-0.1 [18.7 kB] 2024-07-15T06:34:12.598 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../012-librdkafka1_1.8.0-1build1_amd64.deb ... 2024-07-15T06:34:12.598 INFO:teuthology.orchestra.run.smithi089.stdout:Get:89 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-websocket all 1.2.3-1 [34.7 kB] 2024-07-15T06:34:12.600 INFO:teuthology.orchestra.run.smithi089.stdout:Get:90 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-kubernetes all 12.0.1-1ubuntu1 [353 kB] 2024-07-15T06:34:12.608 INFO:teuthology.orchestra.run.smithi089.stdout:Get:91 http://archive.ubuntu.com/ubuntu jammy/main amd64 libonig5 amd64 6.9.7.1-2build1 [172 kB] 2024-07-15T06:34:12.638 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking librdkafka1:amd64 (1.8.0-1build1) ... 2024-07-15T06:34:12.675 INFO:teuthology.orchestra.run.smithi089.stdout:Get:92 http://archive.ubuntu.com/ubuntu jammy/main amd64 libjq1 amd64 1.6-2.1ubuntu3 [133 kB] 2024-07-15T06:34:12.679 INFO:teuthology.orchestra.run.smithi089.stdout:Get:93 http://archive.ubuntu.com/ubuntu jammy/main amd64 jq amd64 1.6-2.1ubuntu3 [52.5 kB] 2024-07-15T06:34:12.681 INFO:teuthology.orchestra.run.smithi089.stdout:Get:94 http://archive.ubuntu.com/ubuntu jammy/main amd64 socat amd64 1.7.4.1-3ubuntu4 [349 kB] 2024-07-15T06:34:12.757 INFO:teuthology.orchestra.run.smithi089.stdout:Get:95 http://archive.ubuntu.com/ubuntu jammy/universe amd64 xmlstarlet amd64 1.6.1-2.1 [265 kB] 2024-07-15T06:34:12.763 INFO:teuthology.orchestra.run.smithi089.stdout:Get:96 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 nvme-cli amd64 1.16-3ubuntu0.1 [474 kB] 2024-07-15T06:34:12.774 INFO:teuthology.orchestra.run.smithi089.stdout:Get:97 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-iniconfig all 1.1.1-2 [6,024 B] 2024-07-15T06:34:12.775 INFO:teuthology.orchestra.run.smithi089.stdout:Get:98 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pastescript all 2.0.2-4 [54.6 kB] 2024-07-15T06:34:12.777 INFO:teuthology.orchestra.run.smithi089.stdout:Get:99 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-pluggy all 0.13.0-7.1 [19.0 kB] 2024-07-15T06:34:12.778 INFO:teuthology.orchestra.run.smithi089.stdout:Get:100 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-psutil amd64 5.9.0-1build1 [158 kB] 2024-07-15T06:34:12.781 INFO:teuthology.orchestra.run.smithi089.stdout:Get:101 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-py all 1.10.0-1 [71.9 kB] 2024-07-15T06:34:12.838 INFO:teuthology.orchestra.run.smithi089.stdout:Get:102 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pygments all 2.11.2+dfsg-2 [750 kB] 2024-07-15T06:34:12.856 INFO:teuthology.orchestra.run.smithi089.stdout:Get:103 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pyinotify all 0.9.6-1.3 [24.8 kB] 2024-07-15T06:34:12.857 INFO:teuthology.orchestra.run.smithi089.stdout:Get:104 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-toml all 0.10.2-1 [16.5 kB] 2024-07-15T06:34:12.919 INFO:teuthology.orchestra.run.smithi089.stdout:Get:105 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-pytest all 6.2.5-1ubuntu2 [214 kB] 2024-07-15T06:34:12.925 INFO:teuthology.orchestra.run.smithi089.stdout:Get:106 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-simplejson amd64 3.17.6-1build1 [54.7 kB] 2024-07-15T06:34:12.926 INFO:teuthology.orchestra.run.smithi089.stdout:Get:107 http://archive.ubuntu.com/ubuntu jammy/universe amd64 qttranslations5-l10n all 5.15.3-1 [1,983 kB] 2024-07-15T06:34:12.999 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package libdouble-conversion3:amd64. 2024-07-15T06:34:13.026 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../013-libdouble-conversion3_3.1.7-4_amd64.deb ... 2024-07-15T06:34:13.066 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking libdouble-conversion3:amd64 (3.1.7-4) ... 2024-07-15T06:34:13.436 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package libpcre2-16-0:amd64. 2024-07-15T06:34:13.462 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../014-libpcre2-16-0_10.39-3ubuntu0.1_amd64.deb ... 2024-07-15T06:34:13.502 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking libpcre2-16-0:amd64 (10.39-3ubuntu0.1) ... 2024-07-15T06:34:13.537 INFO:teuthology.orchestra.run.smithi089.stdout:Fetched 187 MB in 2s (75.7 MB/s) 2024-07-15T06:34:13.675 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package libfmt8:amd64. 2024-07-15T06:34:13.762 INFO:teuthology.orchestra.run.smithi089.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 165982 files and directories currently installed.) 2024-07-15T06:34:13.767 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../000-libfmt8_8.1.1+ds1-2_amd64.deb ... 2024-07-15T06:34:13.849 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking libfmt8:amd64 (8.1.1+ds1-2) ... 2024-07-15T06:34:13.881 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package libqt5core5a:amd64. 2024-07-15T06:34:13.908 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../015-libqt5core5a_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2024-07-15T06:34:13.955 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking libqt5core5a:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-07-15T06:34:14.135 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package liblttng-ust1:amd64. 2024-07-15T06:34:14.162 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../001-liblttng-ust1_2.13.1-1ubuntu1_amd64.deb ... 2024-07-15T06:34:14.202 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking liblttng-ust1:amd64 (2.13.1-1ubuntu1) ... 2024-07-15T06:34:14.376 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package libqt5dbus5:amd64. 2024-07-15T06:34:14.402 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../016-libqt5dbus5_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2024-07-15T06:34:14.442 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking libqt5dbus5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-07-15T06:34:14.472 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package librados2. 2024-07-15T06:34:14.485 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../002-librados2_17.2.7-1jammy_amd64.deb ... 2024-07-15T06:34:14.522 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking librados2 (17.2.7-1jammy) ... 2024-07-15T06:34:14.770 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package libqt5network5:amd64. 2024-07-15T06:34:14.796 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../017-libqt5network5_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2024-07-15T06:34:14.836 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking libqt5network5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-07-15T06:34:15.042 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package librbd1. 2024-07-15T06:34:15.068 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../003-librbd1_17.2.7-1jammy_amd64.deb ... 2024-07-15T06:34:15.108 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking librbd1 (17.2.7-1jammy) ... 2024-07-15T06:34:15.156 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package libthrift-0.16.0:amd64. 2024-07-15T06:34:15.171 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../018-libthrift-0.16.0_0.16.0-2_amd64.deb ... 2024-07-15T06:34:15.214 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking libthrift-0.16.0:amd64 (0.16.0-2) ... 2024-07-15T06:34:15.559 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package librgw2. 2024-07-15T06:34:15.569 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package libcephfs2. 2024-07-15T06:34:15.585 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../019-librgw2_17.2.7-1jammy_amd64.deb ... 2024-07-15T06:34:15.595 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../004-libcephfs2_17.2.7-1jammy_amd64.deb ... 2024-07-15T06:34:15.625 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking librgw2 (17.2.7-1jammy) ... 2024-07-15T06:34:15.636 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking libcephfs2 (17.2.7-1jammy) ... 2024-07-15T06:34:15.905 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-rados. 2024-07-15T06:34:15.920 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../005-python3-rados_17.2.7-1jammy_amd64.deb ... 2024-07-15T06:34:15.955 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-rados (17.2.7-1jammy) ... 2024-07-15T06:34:16.069 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package python3-rgw. 2024-07-15T06:34:16.085 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../020-python3-rgw_17.2.7-1jammy_amd64.deb ... 2024-07-15T06:34:16.119 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking python3-rgw (17.2.7-1jammy) ... 2024-07-15T06:34:16.308 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-ceph-argparse. 2024-07-15T06:34:16.334 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../006-python3-ceph-argparse_17.2.7-1jammy_all.deb ... 2024-07-15T06:34:16.393 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-ceph-argparse (17.2.7-1jammy) ... 2024-07-15T06:34:16.439 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package liboath0:amd64. 2024-07-15T06:34:16.465 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../021-liboath0_2.6.7-3build1_amd64.deb ... 2024-07-15T06:34:16.505 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking liboath0:amd64 (2.6.7-3build1) ... 2024-07-15T06:34:16.627 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-cephfs. 2024-07-15T06:34:16.642 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../007-python3-cephfs_17.2.7-1jammy_amd64.deb ... 2024-07-15T06:34:16.685 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-cephfs (17.2.7-1jammy) ... 2024-07-15T06:34:16.833 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package libradosstriper1. 2024-07-15T06:34:16.860 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../022-libradosstriper1_17.2.7-1jammy_amd64.deb ... 2024-07-15T06:34:16.908 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking libradosstriper1 (17.2.7-1jammy) ... 2024-07-15T06:34:17.005 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-ceph-common. 2024-07-15T06:34:17.031 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../008-python3-ceph-common_17.2.7-1jammy_all.deb ... 2024-07-15T06:34:17.071 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-ceph-common (17.2.7-1jammy) ... 2024-07-15T06:34:17.278 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package ceph-common. 2024-07-15T06:34:17.304 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../023-ceph-common_17.2.7-1jammy_amd64.deb ... 2024-07-15T06:34:17.344 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking ceph-common (17.2.7-1jammy) ... 2024-07-15T06:34:17.374 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-wcwidth. 2024-07-15T06:34:17.400 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../009-python3-wcwidth_0.2.5+dfsg1-1_all.deb ... 2024-07-15T06:34:17.432 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-wcwidth (0.2.5+dfsg1-1) ... 2024-07-15T06:34:17.744 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-prettytable. 2024-07-15T06:34:17.769 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../010-python3-prettytable_2.5.0-2_all.deb ... 2024-07-15T06:34:17.810 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-prettytable (2.5.0-2) ... 2024-07-15T06:34:18.210 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-rbd. 2024-07-15T06:34:18.237 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../011-python3-rbd_17.2.7-1jammy_amd64.deb ... 2024-07-15T06:34:18.338 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-rbd (17.2.7-1jammy) ... 2024-07-15T06:34:18.691 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package librdkafka1:amd64. 2024-07-15T06:34:18.717 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../012-librdkafka1_1.8.0-1build1_amd64.deb ... 2024-07-15T06:34:18.742 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package ceph-base. 2024-07-15T06:34:18.757 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking librdkafka1:amd64 (1.8.0-1build1) ... 2024-07-15T06:34:18.768 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../024-ceph-base_17.2.7-1jammy_amd64.deb ... 2024-07-15T06:34:18.826 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking ceph-base (17.2.7-1jammy) ... 2024-07-15T06:34:19.068 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package libdouble-conversion3:amd64. 2024-07-15T06:34:19.094 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../013-libdouble-conversion3_3.1.7-4_amd64.deb ... 2024-07-15T06:34:19.126 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking libdouble-conversion3:amd64 (3.1.7-4) ... 2024-07-15T06:34:19.472 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package libpcre2-16-0:amd64. 2024-07-15T06:34:19.497 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../014-libpcre2-16-0_10.39-3ubuntu0.1_amd64.deb ... 2024-07-15T06:34:19.505 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package python3-openssl. 2024-07-15T06:34:19.531 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../025-python3-openssl_21.0.0-1_all.deb ... 2024-07-15T06:34:19.538 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking libpcre2-16-0:amd64 (10.39-3ubuntu0.1) ... 2024-07-15T06:34:19.571 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking python3-openssl (21.0.0-1) ... 2024-07-15T06:34:19.798 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package ceph-mgr-modules-core. 2024-07-15T06:34:19.824 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../026-ceph-mgr-modules-core_17.2.7-1jammy_all.deb ... 2024-07-15T06:34:19.856 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking ceph-mgr-modules-core (17.2.7-1jammy) ... 2024-07-15T06:34:20.008 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package libqt5core5a:amd64. 2024-07-15T06:34:20.035 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../015-libqt5core5a_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2024-07-15T06:34:20.083 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking libqt5core5a:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-07-15T06:34:20.220 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package python3-bcrypt. 2024-07-15T06:34:20.246 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../027-python3-bcrypt_3.2.0-1build1_amd64.deb ... 2024-07-15T06:34:20.286 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking python3-bcrypt (3.2.0-1build1) ... 2024-07-15T06:34:20.495 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package libqt5dbus5:amd64. 2024-07-15T06:34:20.521 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../016-libqt5dbus5_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2024-07-15T06:34:20.561 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking libqt5dbus5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-07-15T06:34:20.614 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package python3-jaraco.functools. 2024-07-15T06:34:20.640 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../028-python3-jaraco.functools_3.4.0-2_all.deb ... 2024-07-15T06:34:20.680 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking python3-jaraco.functools (3.4.0-2) ... 2024-07-15T06:34:20.906 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package libqt5network5:amd64. 2024-07-15T06:34:20.933 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../017-libqt5network5_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2024-07-15T06:34:20.967 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package python3-cheroot. 2024-07-15T06:34:20.973 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking libqt5network5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-07-15T06:34:20.993 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../029-python3-cheroot_8.5.2+ds1-1ubuntu3.1_all.deb ... 2024-07-15T06:34:21.042 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking python3-cheroot (8.5.2+ds1-1ubuntu3.1) ... 2024-07-15T06:34:21.276 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package libthrift-0.16.0:amd64. 2024-07-15T06:34:21.302 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../018-libthrift-0.16.0_0.16.0-2_amd64.deb ... 2024-07-15T06:34:21.342 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking libthrift-0.16.0:amd64 (0.16.0-2) ... 2024-07-15T06:34:21.353 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package python3-jaraco.classes. 2024-07-15T06:34:21.379 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../030-python3-jaraco.classes_3.2.1-3_all.deb ... 2024-07-15T06:34:21.419 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking python3-jaraco.classes (3.2.1-3) ... 2024-07-15T06:34:21.670 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package librgw2. 2024-07-15T06:34:21.697 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../019-librgw2_17.2.7-1jammy_amd64.deb ... 2024-07-15T06:34:21.722 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package python3-jaraco.text. 2024-07-15T06:34:21.737 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking librgw2 (17.2.7-1jammy) ... 2024-07-15T06:34:21.749 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../031-python3-jaraco.text_3.6.0-2_all.deb ... 2024-07-15T06:34:21.788 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking python3-jaraco.text (3.6.0-2) ... 2024-07-15T06:34:22.075 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package python3-jaraco.collections. 2024-07-15T06:34:22.101 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../032-python3-jaraco.collections_3.4.0-2_all.deb ... 2024-07-15T06:34:22.141 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking python3-jaraco.collections (3.4.0-2) ... 2024-07-15T06:34:22.190 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-rgw. 2024-07-15T06:34:22.205 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../020-python3-rgw_17.2.7-1jammy_amd64.deb ... 2024-07-15T06:34:22.248 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-rgw (17.2.7-1jammy) ... 2024-07-15T06:34:22.402 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package python3-tempora. 2024-07-15T06:34:22.429 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../033-python3-tempora_4.1.2-1_all.deb ... 2024-07-15T06:34:22.461 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking python3-tempora (4.1.2-1) ... 2024-07-15T06:34:22.534 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package liboath0:amd64. 2024-07-15T06:34:22.549 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../021-liboath0_2.6.7-3build1_amd64.deb ... 2024-07-15T06:34:22.584 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking liboath0:amd64 (2.6.7-3build1) ... 2024-07-15T06:34:22.747 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package python3-portend. 2024-07-15T06:34:22.774 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../034-python3-portend_3.0.0-1_all.deb ... 2024-07-15T06:34:22.813 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking python3-portend (3.0.0-1) ... 2024-07-15T06:34:22.895 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package libradosstriper1. 2024-07-15T06:34:22.921 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../022-libradosstriper1_17.2.7-1jammy_amd64.deb ... 2024-07-15T06:34:22.962 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking libradosstriper1 (17.2.7-1jammy) ... 2024-07-15T06:34:23.116 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package python3-zc.lockfile. 2024-07-15T06:34:23.143 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../035-python3-zc.lockfile_2.0-1_all.deb ... 2024-07-15T06:34:23.183 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking python3-zc.lockfile (2.0-1) ... 2024-07-15T06:34:23.390 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package ceph-common. 2024-07-15T06:34:23.417 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../023-ceph-common_17.2.7-1jammy_amd64.deb ... 2024-07-15T06:34:23.461 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package python3-cherrypy3. 2024-07-15T06:34:23.476 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../036-python3-cherrypy3_18.6.1-4_all.deb ... 2024-07-15T06:34:23.507 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking ceph-common (17.2.7-1jammy) ... 2024-07-15T06:34:23.511 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking python3-cherrypy3 (18.6.1-4) ... 2024-07-15T06:34:23.889 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package python3-natsort. 2024-07-15T06:34:23.916 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../037-python3-natsort_8.0.2-1_all.deb ... 2024-07-15T06:34:23.972 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking python3-natsort (8.0.2-1) ... 2024-07-15T06:34:24.293 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package python3-logutils. 2024-07-15T06:34:24.319 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../038-python3-logutils_0.3.3-8_all.deb ... 2024-07-15T06:34:24.359 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking python3-logutils (0.3.3-8) ... 2024-07-15T06:34:24.603 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package python3-markupsafe. 2024-07-15T06:34:24.618 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../039-python3-markupsafe_2.0.1-2build1_amd64.deb ... 2024-07-15T06:34:24.653 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking python3-markupsafe (2.0.1-2build1) ... 2024-07-15T06:34:24.771 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package ceph-base. 2024-07-15T06:34:24.797 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../024-ceph-base_17.2.7-1jammy_amd64.deb ... 2024-07-15T06:34:24.864 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking ceph-base (17.2.7-1jammy) ... 2024-07-15T06:34:24.939 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package python3-mako. 2024-07-15T06:34:24.966 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../040-python3-mako_1.1.3+ds1-2ubuntu0.1_all.deb ... 2024-07-15T06:34:25.006 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking python3-mako (1.1.3+ds1-2ubuntu0.1) ... 2024-07-15T06:34:25.334 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package python3-simplegeneric. 2024-07-15T06:34:25.361 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../041-python3-simplegeneric_0.8.1-3_all.deb ... 2024-07-15T06:34:25.400 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking python3-simplegeneric (0.8.1-3) ... 2024-07-15T06:34:25.517 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-openssl. 2024-07-15T06:34:25.532 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../025-python3-openssl_21.0.0-1_all.deb ... 2024-07-15T06:34:25.567 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-openssl (21.0.0-1) ... 2024-07-15T06:34:25.687 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package python3-singledispatch. 2024-07-15T06:34:25.713 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../042-python3-singledispatch_3.4.0.3-3_all.deb ... 2024-07-15T06:34:25.753 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking python3-singledispatch (3.4.0.3-3) ... 2024-07-15T06:34:25.828 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package ceph-mgr-modules-core. 2024-07-15T06:34:25.843 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../026-ceph-mgr-modules-core_17.2.7-1jammy_all.deb ... 2024-07-15T06:34:25.886 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking ceph-mgr-modules-core (17.2.7-1jammy) ... 2024-07-15T06:34:26.031 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package python3-webob. 2024-07-15T06:34:26.047 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../043-python3-webob_1%3a1.8.6-1.1_all.deb ... 2024-07-15T06:34:26.081 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking python3-webob (1:1.8.6-1.1) ... 2024-07-15T06:34:26.250 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-bcrypt. 2024-07-15T06:34:26.276 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../027-python3-bcrypt_3.2.0-1build1_amd64.deb ... 2024-07-15T06:34:26.307 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-bcrypt (3.2.0-1build1) ... 2024-07-15T06:34:26.375 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package python3-waitress. 2024-07-15T06:34:26.402 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../044-python3-waitress_1.4.4-1.1ubuntu1_all.deb ... 2024-07-15T06:34:26.437 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking python3-waitress (1.4.4-1.1ubuntu1) ... 2024-07-15T06:34:26.611 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-jaraco.functools. 2024-07-15T06:34:26.637 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../028-python3-jaraco.functools_3.4.0-2_all.deb ... 2024-07-15T06:34:26.677 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-jaraco.functools (3.4.0-2) ... 2024-07-15T06:34:26.745 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package python3-tempita. 2024-07-15T06:34:26.771 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../045-python3-tempita_0.5.2-6ubuntu1_all.deb ... 2024-07-15T06:34:26.811 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking python3-tempita (0.5.2-6ubuntu1) ... 2024-07-15T06:34:26.988 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-cheroot. 2024-07-15T06:34:27.015 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../029-python3-cheroot_8.5.2+ds1-1ubuntu3.1_all.deb ... 2024-07-15T06:34:27.063 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-cheroot (8.5.2+ds1-1ubuntu3.1) ... 2024-07-15T06:34:27.114 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package python3-paste. 2024-07-15T06:34:27.141 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../046-python3-paste_3.5.0+dfsg1-1_all.deb ... 2024-07-15T06:34:27.181 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking python3-paste (3.5.0+dfsg1-1) ... 2024-07-15T06:34:27.391 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-jaraco.classes. 2024-07-15T06:34:27.417 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../030-python3-jaraco.classes_3.2.1-3_all.deb ... 2024-07-15T06:34:27.458 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-jaraco.classes (3.2.1-3) ... 2024-07-15T06:34:27.542 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package python-pastedeploy-tpl. 2024-07-15T06:34:27.569 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../047-python-pastedeploy-tpl_2.1.1-1_all.deb ... 2024-07-15T06:34:27.609 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking python-pastedeploy-tpl (2.1.1-1) ... 2024-07-15T06:34:27.727 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-jaraco.text. 2024-07-15T06:34:27.742 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../031-python3-jaraco.text_3.6.0-2_all.deb ... 2024-07-15T06:34:27.777 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-jaraco.text (3.6.0-2) ... 2024-07-15T06:34:27.920 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package python3-pastedeploy. 2024-07-15T06:34:27.947 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../048-python3-pastedeploy_2.1.1-1_all.deb ... 2024-07-15T06:34:27.986 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking python3-pastedeploy (2.1.1-1) ... 2024-07-15T06:34:28.055 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-jaraco.collections. 2024-07-15T06:34:28.082 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../032-python3-jaraco.collections_3.4.0-2_all.deb ... 2024-07-15T06:34:28.122 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-jaraco.collections (3.4.0-2) ... 2024-07-15T06:34:28.258 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package python3-webtest. 2024-07-15T06:34:28.345 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../049-python3-webtest_2.0.35-1_all.deb ... 2024-07-15T06:34:28.345 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking python3-webtest (2.0.35-1) ... 2024-07-15T06:34:28.425 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-tempora. 2024-07-15T06:34:28.451 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../033-python3-tempora_4.1.2-1_all.deb ... 2024-07-15T06:34:28.491 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-tempora (4.1.2-1) ... 2024-07-15T06:34:28.542 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package python3-pecan. 2024-07-15T06:34:28.557 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../050-python3-pecan_1.3.3-4ubuntu2_all.deb ... 2024-07-15T06:34:28.600 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking python3-pecan (1.3.3-4ubuntu2) ... 2024-07-15T06:34:28.930 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package python3-certifi. 2024-07-15T06:34:28.957 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../051-python3-certifi_2020.6.20-1_all.deb ... 2024-07-15T06:34:28.997 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking python3-certifi (2020.6.20-1) ... 2024-07-15T06:34:29.027 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-portend. 2024-07-15T06:34:29.054 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../034-python3-portend_3.0.0-1_all.deb ... 2024-07-15T06:34:29.102 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-portend (3.0.0-1) ... 2024-07-15T06:34:29.324 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package python3-idna. 2024-07-15T06:34:29.351 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../052-python3-idna_3.3-1ubuntu0.1_all.deb ... 2024-07-15T06:34:29.391 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking python3-idna (3.3-1ubuntu0.1) ... 2024-07-15T06:34:29.441 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-zc.lockfile. 2024-07-15T06:34:29.467 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../035-python3-zc.lockfile_2.0-1_all.deb ... 2024-07-15T06:34:29.507 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-zc.lockfile (2.0-1) ... 2024-07-15T06:34:29.769 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package python3-urllib3. 2024-07-15T06:34:29.795 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../053-python3-urllib3_1.26.5-1~exp1ubuntu0.1_all.deb ... 2024-07-15T06:34:29.844 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking python3-urllib3 (1.26.5-1~exp1ubuntu0.1) ... 2024-07-15T06:34:29.852 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-cherrypy3. 2024-07-15T06:34:29.878 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../036-python3-cherrypy3_18.6.1-4_all.deb ... 2024-07-15T06:34:29.927 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-cherrypy3 (18.6.1-4) ... 2024-07-15T06:34:30.214 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package python3-requests. 2024-07-15T06:34:30.240 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../054-python3-requests_2.25.1+dfsg-2ubuntu0.1_all.deb ... 2024-07-15T06:34:30.280 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-natsort. 2024-07-15T06:34:30.288 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking python3-requests (2.25.1+dfsg-2ubuntu0.1) ... 2024-07-15T06:34:30.306 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../037-python3-natsort_8.0.2-1_all.deb ... 2024-07-15T06:34:30.346 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-natsort (8.0.2-1) ... 2024-07-15T06:34:30.666 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package python3-werkzeug. 2024-07-15T06:34:30.683 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-logutils. 2024-07-15T06:34:30.693 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../055-python3-werkzeug_2.0.2+dfsg1-1ubuntu0.22.04.2_all.deb ... 2024-07-15T06:34:30.710 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../038-python3-logutils_0.3.3-8_all.deb ... 2024-07-15T06:34:30.741 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking python3-werkzeug (2.0.2+dfsg1-1ubuntu0.22.04.2) ... 2024-07-15T06:34:30.750 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-logutils (0.3.3-8) ... 2024-07-15T06:34:31.028 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-markupsafe. 2024-07-15T06:34:31.054 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../039-python3-markupsafe_2.0.1-2build1_amd64.deb ... 2024-07-15T06:34:31.094 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-markupsafe (2.0.1-2build1) ... 2024-07-15T06:34:31.153 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package libsqlite3-mod-ceph. 2024-07-15T06:34:31.179 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../056-libsqlite3-mod-ceph_17.2.7-1jammy_amd64.deb ... 2024-07-15T06:34:31.228 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking libsqlite3-mod-ceph (17.2.7-1jammy) ... 2024-07-15T06:34:31.364 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-mako. 2024-07-15T06:34:31.390 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../040-python3-mako_1.1.3+ds1-2ubuntu0.1_all.deb ... 2024-07-15T06:34:31.430 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-mako (1.1.3+ds1-2ubuntu0.1) ... 2024-07-15T06:34:31.622 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package ceph-mgr. 2024-07-15T06:34:31.649 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../057-ceph-mgr_17.2.7-1jammy_amd64.deb ... 2024-07-15T06:34:31.688 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking ceph-mgr (17.2.7-1jammy) ... 2024-07-15T06:34:31.767 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-simplegeneric. 2024-07-15T06:34:31.793 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../041-python3-simplegeneric_0.8.1-3_all.deb ... 2024-07-15T06:34:31.834 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-simplegeneric (0.8.1-3) ... 2024-07-15T06:34:32.084 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package ceph-mon. 2024-07-15T06:34:32.110 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../058-ceph-mon_17.2.7-1jammy_amd64.deb ... 2024-07-15T06:34:32.137 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-singledispatch. 2024-07-15T06:34:32.150 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking ceph-mon (17.2.7-1jammy) ... 2024-07-15T06:34:32.163 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../042-python3-singledispatch_3.4.0.3-3_all.deb ... 2024-07-15T06:34:32.203 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-singledispatch (3.4.0.3-3) ... 2024-07-15T06:34:32.489 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-webob. 2024-07-15T06:34:32.504 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../043-python3-webob_1%3a1.8.6-1.1_all.deb ... 2024-07-15T06:34:32.539 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-webob (1:1.8.6-1.1) ... 2024-07-15T06:34:32.867 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-waitress. 2024-07-15T06:34:32.883 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../044-python3-waitress_1.4.4-1.1ubuntu1_all.deb ... 2024-07-15T06:34:32.937 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-waitress (1.4.4-1.1ubuntu1) ... 2024-07-15T06:34:33.087 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package ceph-osd. 2024-07-15T06:34:33.114 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../059-ceph-osd_17.2.7-1jammy_amd64.deb ... 2024-07-15T06:34:33.178 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking ceph-osd (17.2.7-1jammy) ... 2024-07-15T06:34:33.203 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-tempita. 2024-07-15T06:34:33.230 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../045-python3-tempita_0.5.2-6ubuntu1_all.deb ... 2024-07-15T06:34:33.270 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-tempita (0.5.2-6ubuntu1) ... 2024-07-15T06:34:33.556 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-paste. 2024-07-15T06:34:33.582 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../046-python3-paste_3.5.0+dfsg1-1_all.deb ... 2024-07-15T06:34:33.623 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-paste (3.5.0+dfsg1-1) ... 2024-07-15T06:34:33.960 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python-pastedeploy-tpl. 2024-07-15T06:34:33.986 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../047-python-pastedeploy-tpl_2.1.1-1_all.deb ... 2024-07-15T06:34:34.017 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python-pastedeploy-tpl (2.1.1-1) ... 2024-07-15T06:34:34.354 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-pastedeploy. 2024-07-15T06:34:34.381 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../048-python3-pastedeploy_2.1.1-1_all.deb ... 2024-07-15T06:34:34.420 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-pastedeploy (2.1.1-1) ... 2024-07-15T06:34:34.723 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-webtest. 2024-07-15T06:34:34.750 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../049-python3-webtest_2.0.35-1_all.deb ... 2024-07-15T06:34:34.790 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-webtest (2.0.35-1) ... 2024-07-15T06:34:34.874 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package ceph. 2024-07-15T06:34:34.900 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../060-ceph_17.2.7-1jammy_amd64.deb ... 2024-07-15T06:34:34.932 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking ceph (17.2.7-1jammy) ... 2024-07-15T06:34:35.101 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-pecan. 2024-07-15T06:34:35.128 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../050-python3-pecan_1.3.3-4ubuntu2_all.deb ... 2024-07-15T06:34:35.168 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-pecan (1.3.3-4ubuntu2) ... 2024-07-15T06:34:35.226 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package ceph-fuse. 2024-07-15T06:34:35.253 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../061-ceph-fuse_17.2.7-1jammy_amd64.deb ... 2024-07-15T06:34:35.301 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking ceph-fuse (17.2.7-1jammy) ... 2024-07-15T06:34:35.539 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-certifi. 2024-07-15T06:34:35.566 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../051-python3-certifi_2020.6.20-1_all.deb ... 2024-07-15T06:34:35.606 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-certifi (2020.6.20-1) ... 2024-07-15T06:34:35.663 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package ceph-mds. 2024-07-15T06:34:35.689 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../062-ceph-mds_17.2.7-1jammy_amd64.deb ... 2024-07-15T06:34:35.729 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking ceph-mds (17.2.7-1jammy) ... 2024-07-15T06:34:35.909 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-idna. 2024-07-15T06:34:35.924 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../052-python3-idna_3.3-1ubuntu0.1_all.deb ... 2024-07-15T06:34:35.959 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-idna (3.3-1ubuntu0.1) ... 2024-07-15T06:34:36.099 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package cephadm. 2024-07-15T06:34:36.125 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../063-cephadm_17.2.7-1jammy_amd64.deb ... 2024-07-15T06:34:36.157 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking cephadm (17.2.7-1jammy) ... 2024-07-15T06:34:36.287 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-urllib3. 2024-07-15T06:34:36.313 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../053-python3-urllib3_1.26.5-1~exp1ubuntu0.1_all.deb ... 2024-07-15T06:34:36.353 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-urllib3 (1.26.5-1~exp1ubuntu0.1) ... 2024-07-15T06:34:36.426 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package python-babel-localedata. 2024-07-15T06:34:36.453 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../064-python-babel-localedata_2.8.0+dfsg.1-7_all.deb ... 2024-07-15T06:34:36.493 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking python-babel-localedata (2.8.0+dfsg.1-7) ... 2024-07-15T06:34:36.673 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-requests. 2024-07-15T06:34:36.700 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../054-python3-requests_2.25.1+dfsg-2ubuntu0.1_all.deb ... 2024-07-15T06:34:36.740 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-requests (2.25.1+dfsg-2ubuntu0.1) ... 2024-07-15T06:34:37.035 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-werkzeug. 2024-07-15T06:34:37.061 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../055-python3-werkzeug_2.0.2+dfsg1-1ubuntu0.22.04.2_all.deb ... 2024-07-15T06:34:37.101 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-werkzeug (2.0.2+dfsg1-1ubuntu0.22.04.2) ... 2024-07-15T06:34:37.455 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package libsqlite3-mod-ceph. 2024-07-15T06:34:37.481 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../056-libsqlite3-mod-ceph_17.2.7-1jammy_amd64.deb ... 2024-07-15T06:34:37.521 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking libsqlite3-mod-ceph (17.2.7-1jammy) ... 2024-07-15T06:34:37.540 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package python3-babel. 2024-07-15T06:34:37.566 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../065-python3-babel_2.8.0+dfsg.1-7_all.deb ... 2024-07-15T06:34:37.606 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking python3-babel (2.8.0+dfsg.1-7) ... 2024-07-15T06:34:37.891 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package ceph-mgr. 2024-07-15T06:34:37.917 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../057-ceph-mgr_17.2.7-1jammy_amd64.deb ... 2024-07-15T06:34:37.934 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package python3-jinja2. 2024-07-15T06:34:37.957 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking ceph-mgr (17.2.7-1jammy) ... 2024-07-15T06:34:37.960 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../066-python3-jinja2_3.0.3-1ubuntu0.2_all.deb ... 2024-07-15T06:34:38.000 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking python3-jinja2 (3.0.3-1ubuntu0.2) ... 2024-07-15T06:34:38.335 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package ceph-mon. 2024-07-15T06:34:38.362 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../058-ceph-mon_17.2.7-1jammy_amd64.deb ... 2024-07-15T06:34:38.393 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking ceph-mon (17.2.7-1jammy) ... 2024-07-15T06:34:39.038 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package ceph-mgr-cephadm. 2024-07-15T06:34:39.065 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../067-ceph-mgr-cephadm_17.2.7-1jammy_all.deb ... 2024-07-15T06:34:39.137 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking ceph-mgr-cephadm (17.2.7-1jammy) ... 2024-07-15T06:34:39.422 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package ceph-osd. 2024-07-15T06:34:39.441 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package python3-repoze.lru. 2024-07-15T06:34:39.448 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../059-ceph-osd_17.2.7-1jammy_amd64.deb ... 2024-07-15T06:34:39.468 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../068-python3-repoze.lru_0.7-2_all.deb ... 2024-07-15T06:34:39.488 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking ceph-osd (17.2.7-1jammy) ... 2024-07-15T06:34:39.507 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking python3-repoze.lru (0.7-2) ... 2024-07-15T06:34:39.777 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package python3-routes. 2024-07-15T06:34:39.804 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../069-python3-routes_2.5.1-1ubuntu1_all.deb ... 2024-07-15T06:34:39.843 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking python3-routes (2.5.1-1ubuntu1) ... 2024-07-15T06:34:40.130 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package ceph-mgr-dashboard. 2024-07-15T06:34:40.157 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../070-ceph-mgr-dashboard_17.2.7-1jammy_all.deb ... 2024-07-15T06:34:40.188 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking ceph-mgr-dashboard (17.2.7-1jammy) ... 2024-07-15T06:34:40.625 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package python3-sklearn-lib:amd64. 2024-07-15T06:34:40.646 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../071-python3-sklearn-lib_0.23.2-5ubuntu6_amd64.deb ... 2024-07-15T06:34:40.683 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking python3-sklearn-lib:amd64 (0.23.2-5ubuntu6) ... 2024-07-15T06:34:41.195 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package python3-joblib. 2024-07-15T06:34:41.217 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package ceph. 2024-07-15T06:34:41.221 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../072-python3-joblib_0.17.0-4ubuntu1_all.deb ... 2024-07-15T06:34:41.243 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../060-ceph_17.2.7-1jammy_amd64.deb ... 2024-07-15T06:34:41.261 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking python3-joblib (0.17.0-4ubuntu1) ... 2024-07-15T06:34:41.292 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking ceph (17.2.7-1jammy) ... 2024-07-15T06:34:41.623 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package python3-threadpoolctl. 2024-07-15T06:34:41.642 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../073-python3-threadpoolctl_3.1.0-1_all.deb ... 2024-07-15T06:34:41.681 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking python3-threadpoolctl (3.1.0-1) ... 2024-07-15T06:34:41.711 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package ceph-fuse. 2024-07-15T06:34:41.738 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../061-ceph-fuse_17.2.7-1jammy_amd64.deb ... 2024-07-15T06:34:41.786 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking ceph-fuse (17.2.7-1jammy) ... 2024-07-15T06:34:41.993 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package python3-sklearn. 2024-07-15T06:34:42.019 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../074-python3-sklearn_0.23.2-5ubuntu6_all.deb ... 2024-07-15T06:34:42.059 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking python3-sklearn (0.23.2-5ubuntu6) ... 2024-07-15T06:34:42.173 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package ceph-mds. 2024-07-15T06:34:42.199 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../062-ceph-mds_17.2.7-1jammy_amd64.deb ... 2024-07-15T06:34:42.239 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking ceph-mds (17.2.7-1jammy) ... 2024-07-15T06:34:42.634 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package cephadm. 2024-07-15T06:34:42.661 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../063-cephadm_17.2.7-1jammy_amd64.deb ... 2024-07-15T06:34:42.701 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking cephadm (17.2.7-1jammy) ... 2024-07-15T06:34:42.801 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package ceph-mgr-diskprediction-local. 2024-07-15T06:34:42.828 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../075-ceph-mgr-diskprediction-local_17.2.7-1jammy_all.deb ... 2024-07-15T06:34:42.867 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking ceph-mgr-diskprediction-local (17.2.7-1jammy) ... 2024-07-15T06:34:42.995 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python-babel-localedata. 2024-07-15T06:34:43.010 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../064-python-babel-localedata_2.8.0+dfsg.1-7_all.deb ... 2024-07-15T06:34:43.045 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python-babel-localedata (2.8.0+dfsg.1-7) ... 2024-07-15T06:34:43.804 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package python3-cachetools. 2024-07-15T06:34:43.820 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../076-python3-cachetools_5.0.0-1_all.deb ... 2024-07-15T06:34:43.854 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking python3-cachetools (5.0.0-1) ... 2024-07-15T06:34:44.075 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-babel. 2024-07-15T06:34:44.102 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../065-python3-babel_2.8.0+dfsg.1-7_all.deb ... 2024-07-15T06:34:44.115 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package python3-pyasn1. 2024-07-15T06:34:44.141 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-babel (2.8.0+dfsg.1-7) ... 2024-07-15T06:34:44.142 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../077-python3-pyasn1_0.4.8-1_all.deb ... 2024-07-15T06:34:44.173 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking python3-pyasn1 (0.4.8-1) ... 2024-07-15T06:34:44.476 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package python3-pyasn1-modules. 2024-07-15T06:34:44.503 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../078-python3-pyasn1-modules_0.2.1-1_all.deb ... 2024-07-15T06:34:44.551 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking python3-pyasn1-modules (0.2.1-1) ... 2024-07-15T06:34:44.854 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package python3-rsa. 2024-07-15T06:34:44.881 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../079-python3-rsa_4.8-1_all.deb ... 2024-07-15T06:34:44.921 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking python3-rsa (4.8-1) ... 2024-07-15T06:34:45.179 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-jinja2. 2024-07-15T06:34:45.206 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../066-python3-jinja2_3.0.3-1ubuntu0.2_all.deb ... 2024-07-15T06:34:45.249 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package python3-google-auth. 2024-07-15T06:34:45.276 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../080-python3-google-auth_1.5.1-3_all.deb ... 2024-07-15T06:34:45.316 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking python3-google-auth (1.5.1-3) ... 2024-07-15T06:34:45.562 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-jinja2 (3.0.3-1ubuntu0.2) ... 2024-07-15T06:34:45.669 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package python3-requests-oauthlib. 2024-07-15T06:34:45.696 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../081-python3-requests-oauthlib_1.3.0+ds-0.1_all.deb ... 2024-07-15T06:34:45.735 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking python3-requests-oauthlib (1.3.0+ds-0.1) ... 2024-07-15T06:34:45.924 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package ceph-mgr-cephadm. 2024-07-15T06:34:45.950 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../067-ceph-mgr-cephadm_17.2.7-1jammy_all.deb ... 2024-07-15T06:34:45.990 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking ceph-mgr-cephadm (17.2.7-1jammy) ... 2024-07-15T06:34:46.013 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package python3-websocket. 2024-07-15T06:34:46.037 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../082-python3-websocket_1.2.3-1_all.deb ... 2024-07-15T06:34:46.071 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking python3-websocket (1.2.3-1) ... 2024-07-15T06:34:46.335 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-repoze.lru. 2024-07-15T06:34:46.362 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../068-python3-repoze.lru_0.7-2_all.deb ... 2024-07-15T06:34:46.402 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-repoze.lru (0.7-2) ... 2024-07-15T06:34:46.460 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package python3-kubernetes. 2024-07-15T06:34:46.486 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../083-python3-kubernetes_12.0.1-1ubuntu1_all.deb ... 2024-07-15T06:34:46.526 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking python3-kubernetes (12.0.1-1ubuntu1) ... 2024-07-15T06:34:46.671 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-routes. 2024-07-15T06:34:46.698 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../069-python3-routes_2.5.1-1ubuntu1_all.deb ... 2024-07-15T06:34:46.729 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-routes (2.5.1-1ubuntu1) ... 2024-07-15T06:34:47.075 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package ceph-mgr-dashboard. 2024-07-15T06:34:47.101 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../070-ceph-mgr-dashboard_17.2.7-1jammy_all.deb ... 2024-07-15T06:34:47.131 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package ceph-mgr-k8sevents. 2024-07-15T06:34:47.141 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking ceph-mgr-dashboard (17.2.7-1jammy) ... 2024-07-15T06:34:47.159 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../084-ceph-mgr-k8sevents_17.2.7-1jammy_all.deb ... 2024-07-15T06:34:47.198 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking ceph-mgr-k8sevents (17.2.7-1jammy) ... 2024-07-15T06:34:47.476 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package libonig5:amd64. 2024-07-15T06:34:47.503 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../085-libonig5_6.9.7.1-2build1_amd64.deb ... 2024-07-15T06:34:47.534 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking libonig5:amd64 (6.9.7.1-2build1) ... 2024-07-15T06:34:47.544 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-sklearn-lib:amd64. 2024-07-15T06:34:47.560 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../071-python3-sklearn-lib_0.23.2-5ubuntu6_amd64.deb ... 2024-07-15T06:34:47.603 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-sklearn-lib:amd64 (0.23.2-5ubuntu6) ... 2024-07-15T06:34:47.871 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package libjq1:amd64. 2024-07-15T06:34:47.898 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../086-libjq1_1.6-2.1ubuntu3_amd64.deb ... 2024-07-15T06:34:47.937 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking libjq1:amd64 (1.6-2.1ubuntu3) ... 2024-07-15T06:34:48.107 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-joblib. 2024-07-15T06:34:48.134 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../072-python3-joblib_0.17.0-4ubuntu1_all.deb ... 2024-07-15T06:34:48.173 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-joblib (0.17.0-4ubuntu1) ... 2024-07-15T06:34:48.197 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package jq. 2024-07-15T06:34:48.213 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../087-jq_1.6-2.1ubuntu3_amd64.deb ... 2024-07-15T06:34:48.256 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking jq (1.6-2.1ubuntu3) ... 2024-07-15T06:34:48.477 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-threadpoolctl. 2024-07-15T06:34:48.490 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../073-python3-threadpoolctl_3.1.0-1_all.deb ... 2024-07-15T06:34:48.501 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package socat. 2024-07-15T06:34:48.528 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../088-socat_1.7.4.1-3ubuntu4_amd64.deb ... 2024-07-15T06:34:48.535 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-threadpoolctl (3.1.0-1) ... 2024-07-15T06:34:48.567 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking socat (1.7.4.1-3ubuntu4) ... 2024-07-15T06:34:48.838 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-sklearn. 2024-07-15T06:34:48.863 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package xmlstarlet. 2024-07-15T06:34:48.865 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../074-python3-sklearn_0.23.2-5ubuntu6_all.deb ... 2024-07-15T06:34:48.878 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../089-xmlstarlet_1.6.1-2.1_amd64.deb ... 2024-07-15T06:34:48.904 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-sklearn (0.23.2-5ubuntu6) ... 2024-07-15T06:34:48.920 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking xmlstarlet (1.6.1-2.1) ... 2024-07-15T06:34:49.308 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package ceph-test. 2024-07-15T06:34:49.335 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../090-ceph-test_17.2.7-1jammy_amd64.deb ... 2024-07-15T06:34:49.374 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking ceph-test (17.2.7-1jammy) ... 2024-07-15T06:34:49.630 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package ceph-mgr-diskprediction-local. 2024-07-15T06:34:49.657 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../075-ceph-mgr-diskprediction-local_17.2.7-1jammy_all.deb ... 2024-07-15T06:34:49.696 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking ceph-mgr-diskprediction-local (17.2.7-1jammy) ... 2024-07-15T06:34:50.692 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-cachetools. 2024-07-15T06:34:50.718 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../076-python3-cachetools_5.0.0-1_all.deb ... 2024-07-15T06:34:50.758 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-cachetools (5.0.0-1) ... 2024-07-15T06:34:51.044 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-pyasn1. 2024-07-15T06:34:51.071 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../077-python3-pyasn1_0.4.8-1_all.deb ... 2024-07-15T06:34:51.110 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-pyasn1 (0.4.8-1) ... 2024-07-15T06:34:51.456 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-pyasn1-modules. 2024-07-15T06:34:51.482 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../078-python3-pyasn1-modules_0.2.1-1_all.deb ... 2024-07-15T06:34:51.522 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-pyasn1-modules (0.2.1-1) ... 2024-07-15T06:34:51.813 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package ceph-volume. 2024-07-15T06:34:51.834 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-rsa. 2024-07-15T06:34:51.837 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../091-ceph-volume_17.2.7-1jammy_all.deb ... 2024-07-15T06:34:51.861 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../079-python3-rsa_4.8-1_all.deb ... 2024-07-15T06:34:51.879 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking ceph-volume (17.2.7-1jammy) ... 2024-07-15T06:34:51.900 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-rsa (4.8-1) ... 2024-07-15T06:34:52.187 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-google-auth. 2024-07-15T06:34:52.214 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../080-python3-google-auth_1.5.1-3_all.deb ... 2024-07-15T06:34:52.226 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package libcephfs-dev. 2024-07-15T06:34:52.242 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../092-libcephfs-dev_17.2.7-1jammy_amd64.deb ... 2024-07-15T06:34:52.245 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-google-auth (1.5.1-3) ... 2024-07-15T06:34:52.275 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking libcephfs-dev (17.2.7-1jammy) ... 2024-07-15T06:34:52.573 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-requests-oauthlib. 2024-07-15T06:34:52.600 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../081-python3-requests-oauthlib_1.3.0+ds-0.1_all.deb ... 2024-07-15T06:34:52.640 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-requests-oauthlib (1.3.0+ds-0.1) ... 2024-07-15T06:34:52.688 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package nvme-cli. 2024-07-15T06:34:52.715 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../093-nvme-cli_1.16-3ubuntu0.1_amd64.deb ... 2024-07-15T06:34:52.754 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking nvme-cli (1.16-3ubuntu0.1) ... 2024-07-15T06:34:52.968 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-websocket. 2024-07-15T06:34:52.995 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../082-python3-websocket_1.2.3-1_all.deb ... 2024-07-15T06:34:53.034 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-websocket (1.2.3-1) ... 2024-07-15T06:34:53.134 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package python3-iniconfig. 2024-07-15T06:34:53.151 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../094-python3-iniconfig_1.1.1-2_all.deb ... 2024-07-15T06:34:53.193 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking python3-iniconfig (1.1.1-2) ... 2024-07-15T06:34:53.423 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-kubernetes. 2024-07-15T06:34:53.450 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../083-python3-kubernetes_12.0.1-1ubuntu1_all.deb ... 2024-07-15T06:34:53.462 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package python3-pastescript. 2024-07-15T06:34:53.480 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-kubernetes (12.0.1-1ubuntu1) ... 2024-07-15T06:34:53.490 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../095-python3-pastescript_2.0.2-4_all.deb ... 2024-07-15T06:34:53.529 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking python3-pastescript (2.0.2-4) ... 2024-07-15T06:34:53.832 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package python3-pluggy. 2024-07-15T06:34:53.860 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../096-python3-pluggy_0.13.0-7.1_all.deb ... 2024-07-15T06:34:53.899 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking python3-pluggy (0.13.0-7.1) ... 2024-07-15T06:34:54.161 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package ceph-mgr-k8sevents. 2024-07-15T06:34:54.177 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../084-ceph-mgr-k8sevents_17.2.7-1jammy_all.deb ... 2024-07-15T06:34:54.210 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package python3-psutil. 2024-07-15T06:34:54.219 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking ceph-mgr-k8sevents (17.2.7-1jammy) ... 2024-07-15T06:34:54.238 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../097-python3-psutil_5.9.0-1build1_amd64.deb ... 2024-07-15T06:34:54.276 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking python3-psutil (5.9.0-1build1) ... 2024-07-15T06:34:54.539 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package libonig5:amd64. 2024-07-15T06:34:54.555 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../085-libonig5_6.9.7.1-2build1_amd64.deb ... 2024-07-15T06:34:54.597 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking libonig5:amd64 (6.9.7.1-2build1) ... 2024-07-15T06:34:54.630 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package python3-py. 2024-07-15T06:34:54.658 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../098-python3-py_1.10.0-1_all.deb ... 2024-07-15T06:34:54.705 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking python3-py (1.10.0-1) ... 2024-07-15T06:34:54.933 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package libjq1:amd64. 2024-07-15T06:34:54.960 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../086-libjq1_1.6-2.1ubuntu3_amd64.deb ... 2024-07-15T06:34:55.008 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package python3-pygments. 2024-07-15T06:34:55.036 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../099-python3-pygments_2.11.2+dfsg-2_all.deb ... 2024-07-15T06:34:55.074 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking python3-pygments (2.11.2+dfsg-2) ... 2024-07-15T06:34:55.125 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking libjq1:amd64 (1.6-2.1ubuntu3) ... 2024-07-15T06:34:55.395 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package jq. 2024-07-15T06:34:55.410 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../087-jq_1.6-2.1ubuntu3_amd64.deb ... 2024-07-15T06:34:55.453 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking jq (1.6-2.1ubuntu3) ... 2024-07-15T06:34:55.537 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package python3-pyinotify. 2024-07-15T06:34:55.565 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../100-python3-pyinotify_0.9.6-1.3_all.deb ... 2024-07-15T06:34:55.603 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking python3-pyinotify (0.9.6-1.3) ... 2024-07-15T06:34:55.731 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package socat. 2024-07-15T06:34:55.758 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../088-socat_1.7.4.1-3ubuntu4_amd64.deb ... 2024-07-15T06:34:55.797 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking socat (1.7.4.1-3ubuntu4) ... 2024-07-15T06:34:55.923 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package python3-toml. 2024-07-15T06:34:55.951 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../101-python3-toml_0.10.2-1_all.deb ... 2024-07-15T06:34:55.989 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking python3-toml (0.10.2-1) ... 2024-07-15T06:34:56.117 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package xmlstarlet. 2024-07-15T06:34:56.133 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../089-xmlstarlet_1.6.1-2.1_amd64.deb ... 2024-07-15T06:34:56.176 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking xmlstarlet (1.6.1-2.1) ... 2024-07-15T06:34:56.317 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package python3-pytest. 2024-07-15T06:34:56.345 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../102-python3-pytest_6.2.5-1ubuntu2_all.deb ... 2024-07-15T06:34:56.383 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking python3-pytest (6.2.5-1ubuntu2) ... 2024-07-15T06:34:56.530 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package ceph-test. 2024-07-15T06:34:56.546 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../090-ceph-test_17.2.7-1jammy_amd64.deb ... 2024-07-15T06:34:56.588 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking ceph-test (17.2.7-1jammy) ... 2024-07-15T06:34:56.729 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package python3-simplejson. 2024-07-15T06:34:56.757 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../103-python3-simplejson_3.17.6-1build1_amd64.deb ... 2024-07-15T06:34:56.795 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking python3-simplejson (3.17.6-1build1) ... 2024-07-15T06:34:57.073 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package qttranslations5-l10n. 2024-07-15T06:34:57.101 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../104-qttranslations5-l10n_5.15.3-1_all.deb ... 2024-07-15T06:34:57.140 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking qttranslations5-l10n (5.15.3-1) ... 2024-07-15T06:34:57.952 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package radosgw. 2024-07-15T06:34:57.980 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../105-radosgw_17.2.7-1jammy_amd64.deb ... 2024-07-15T06:34:58.019 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking radosgw (17.2.7-1jammy) ... 2024-07-15T06:34:58.914 INFO:teuthology.orchestra.run.smithi134.stdout:Selecting previously unselected package rbd-fuse. 2024-07-15T06:34:58.930 INFO:teuthology.orchestra.run.smithi134.stdout:Preparing to unpack .../106-rbd-fuse_17.2.7-1jammy_amd64.deb ... 2024-07-15T06:34:58.963 INFO:teuthology.orchestra.run.smithi134.stdout:Unpacking rbd-fuse (17.2.7-1jammy) ... 2024-07-15T06:34:59.075 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package ceph-volume. 2024-07-15T06:34:59.091 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../091-ceph-volume_17.2.7-1jammy_all.deb ... 2024-07-15T06:34:59.133 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking ceph-volume (17.2.7-1jammy) ... 2024-07-15T06:34:59.267 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up python3-iniconfig (1.1.1-2) ... 2024-07-15T06:34:59.471 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package libcephfs-dev. 2024-07-15T06:34:59.487 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../092-libcephfs-dev_17.2.7-1jammy_amd64.deb ... 2024-07-15T06:34:59.602 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up libdouble-conversion3:amd64 (3.1.7-4) ... 2024-07-15T06:34:59.648 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking libcephfs-dev (17.2.7-1jammy) ... 2024-07-15T06:34:59.727 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up nvme-cli (1.16-3ubuntu0.1) ... 2024-07-15T06:35:00.003 INFO:teuthology.orchestra.run.smithi134.stdout:Created symlink /etc/systemd/system/default.target.wants/nvmefc-boot-connections.service -> /lib/systemd/system/nvmefc-boot-connections.service. 2024-07-15T06:35:00.033 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package nvme-cli. 2024-07-15T06:35:00.046 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../093-nvme-cli_1.16-3ubuntu0.1_amd64.deb ... 2024-07-15T06:35:00.099 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking nvme-cli (1.16-3ubuntu0.1) ... 2024-07-15T06:35:00.359 INFO:teuthology.orchestra.run.smithi134.stdout:Created symlink /etc/systemd/system/default.target.wants/nvmf-autoconnect.service -> /lib/systemd/system/nvmf-autoconnect.service. 2024-07-15T06:35:00.497 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-iniconfig. 2024-07-15T06:35:00.524 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../094-python3-iniconfig_1.1.1-2_all.deb ... 2024-07-15T06:35:00.563 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-iniconfig (1.1.1-2) ... 2024-07-15T06:35:00.803 INFO:teuthology.orchestra.run.smithi134.stdout:nvmf-connect.target is a disabled or a static unit, not starting it. 2024-07-15T06:35:00.833 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-pastescript. 2024-07-15T06:35:00.860 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../095-python3-pastescript_2.0.2-4_all.deb ... 2024-07-15T06:35:00.871 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up cephadm (17.2.7-1jammy) ... 2024-07-15T06:35:00.891 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-pastescript (2.0.2-4) ... 2024-07-15T06:35:01.169 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-pluggy. 2024-07-15T06:35:01.185 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../096-python3-pluggy_0.13.0-7.1_all.deb ... 2024-07-15T06:35:01.227 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-pluggy (0.13.0-7.1) ... 2024-07-15T06:35:01.440 INFO:teuthology.orchestra.run.smithi134.stdout:Adding system user cephadm....done 2024-07-15T06:35:01.539 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-psutil. 2024-07-15T06:35:01.566 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../097-python3-psutil_5.9.0-1build1_amd64.deb ... 2024-07-15T06:35:01.577 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up python3-waitress (1.4.4-1.1ubuntu1) ... 2024-07-15T06:35:01.605 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-psutil (5.9.0-1build1) ... 2024-07-15T06:35:01.846 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up python3-jaraco.classes (3.2.1-3) ... 2024-07-15T06:35:01.984 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-py. 2024-07-15T06:35:02.011 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../098-python3-py_1.10.0-1_all.deb ... 2024-07-15T06:35:02.050 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-py (1.10.0-1) ... 2024-07-15T06:35:02.088 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up python3-jaraco.functools (3.4.0-2) ... 2024-07-15T06:35:02.340 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up python3-repoze.lru (0.7-2) ... 2024-07-15T06:35:02.379 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-pygments. 2024-07-15T06:35:02.406 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../099-python3-pygments_2.11.2+dfsg-2_all.deb ... 2024-07-15T06:35:02.445 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-pygments (2.11.2+dfsg-2) ... 2024-07-15T06:35:02.649 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up liboath0:amd64 (2.6.7-3build1) ... 2024-07-15T06:35:02.775 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up python3-py (1.10.0-1) ... 2024-07-15T06:35:02.949 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-pyinotify. 2024-07-15T06:35:02.976 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../100-python3-pyinotify_0.9.6-1.3_all.deb ... 2024-07-15T06:35:03.015 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-pyinotify (0.9.6-1.3) ... 2024-07-15T06:35:03.094 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up python3-joblib (0.17.0-4ubuntu1) ... 2024-07-15T06:35:03.310 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-toml. 2024-07-15T06:35:03.337 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../101-python3-toml_0.10.2-1_all.deb ... 2024-07-15T06:35:03.377 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-toml (0.10.2-1) ... 2024-07-15T06:35:03.463 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up python3-cachetools (5.0.0-1) ... 2024-07-15T06:35:03.613 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-pytest. 2024-07-15T06:35:03.628 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../102-python3-pytest_6.2.5-1ubuntu2_all.deb ... 2024-07-15T06:35:03.671 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-pytest (6.2.5-1ubuntu2) ... 2024-07-15T06:35:03.739 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up python3-openssl (21.0.0-1) ... 2024-07-15T06:35:04.006 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up python3-bcrypt (3.2.0-1build1) ... 2024-07-15T06:35:04.016 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-simplejson. 2024-07-15T06:35:04.044 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../103-python3-simplejson_3.17.6-1build1_amd64.deb ... 2024-07-15T06:35:04.082 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-simplejson (3.17.6-1build1) ... 2024-07-15T06:35:04.266 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up python3-pyinotify (0.9.6-1.3) ... 2024-07-15T06:35:04.352 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package qttranslations5-l10n. 2024-07-15T06:35:04.367 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../104-qttranslations5-l10n_5.15.3-1_all.deb ... 2024-07-15T06:35:04.402 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking qttranslations5-l10n (5.15.3-1) ... 2024-07-15T06:35:04.525 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up python3-threadpoolctl (3.1.0-1) ... 2024-07-15T06:35:04.802 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up python3-ceph-argparse (17.2.7-1jammy) ... 2024-07-15T06:35:05.061 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up python3-sklearn-lib:amd64 (0.23.2-5ubuntu6) ... 2024-07-15T06:35:05.098 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package radosgw. 2024-07-15T06:35:05.125 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../105-radosgw_17.2.7-1jammy_amd64.deb ... 2024-07-15T06:35:05.155 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking radosgw (17.2.7-1jammy) ... 2024-07-15T06:35:05.295 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up python3-markupsafe (2.0.1-2build1) ... 2024-07-15T06:35:05.563 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up libpcre2-16-0:amd64 (10.39-3ubuntu0.1) ... 2024-07-15T06:35:05.673 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up python3-psutil (5.9.0-1build1) ... 2024-07-15T06:35:06.024 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up python-babel-localedata (2.8.0+dfsg.1-7) ... 2024-07-15T06:35:06.100 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package rbd-fuse. 2024-07-15T06:35:06.116 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../106-rbd-fuse_17.2.7-1jammy_amd64.deb ... 2024-07-15T06:35:06.150 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up python3-natsort (8.0.2-1) ... 2024-07-15T06:35:06.159 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking rbd-fuse (17.2.7-1jammy) ... 2024-07-15T06:35:06.418 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up python3-routes (2.5.1-1ubuntu1) ... 2024-07-15T06:35:06.487 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-iniconfig (1.1.1-2) ... 2024-07-15T06:35:06.694 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up python3-simplejson (3.17.6-1build1) ... 2024-07-15T06:35:06.738 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up libdouble-conversion3:amd64 (3.1.7-4) ... 2024-07-15T06:35:06.864 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up nvme-cli (1.16-3ubuntu0.1) ... 2024-07-15T06:35:07.012 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up python3-pygments (2.11.2+dfsg-2) ... 2024-07-15T06:35:07.203 INFO:teuthology.orchestra.run.smithi089.stdout:Created symlink /etc/systemd/system/default.target.wants/nvmefc-boot-connections.service -> /lib/systemd/system/nvmefc-boot-connections.service. 2024-07-15T06:35:07.516 INFO:teuthology.orchestra.run.smithi089.stdout:Created symlink /etc/systemd/system/default.target.wants/nvmf-autoconnect.service -> /lib/systemd/system/nvmf-autoconnect.service. 2024-07-15T06:35:07.647 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up python3-tempita (0.5.2-6ubuntu1) ... 2024-07-15T06:35:07.907 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up python-pastedeploy-tpl (2.1.1-1) ... 2024-07-15T06:35:07.971 INFO:teuthology.orchestra.run.smithi089.stdout:nvmf-connect.target is a disabled or a static unit, not starting it. 2024-07-15T06:35:08.033 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up qttranslations5-l10n (5.15.3-1) ... 2024-07-15T06:35:08.050 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up cephadm (17.2.7-1jammy) ... 2024-07-15T06:35:08.159 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up python3-wcwidth (0.2.5+dfsg1-1) ... 2024-07-15T06:35:08.452 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up python3-certifi (2020.6.20-1) ... 2024-07-15T06:35:08.619 INFO:teuthology.orchestra.run.smithi089.stdout:Adding system user cephadm....done 2024-07-15T06:35:08.711 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up python3-paste (3.5.0+dfsg1-1) ... 2024-07-15T06:35:08.739 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-waitress (1.4.4-1.1ubuntu1) ... 2024-07-15T06:35:09.016 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-jaraco.classes (3.2.1-3) ... 2024-07-15T06:35:09.105 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up python3-cheroot (8.5.2+ds1-1ubuntu3.1) ... 2024-07-15T06:35:09.284 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-jaraco.functools (3.4.0-2) ... 2024-07-15T06:35:09.381 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up python3-werkzeug (2.0.2+dfsg1-1ubuntu0.22.04.2) ... 2024-07-15T06:35:09.544 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-repoze.lru (0.7-2) ... 2024-07-15T06:35:09.732 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up python3-jaraco.text (3.6.0-2) ... 2024-07-15T06:35:09.795 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up liboath0:amd64 (2.6.7-3build1) ... 2024-07-15T06:35:09.921 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-py (1.10.0-1) ... 2024-07-15T06:35:09.983 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up socat (1.7.4.1-3ubuntu4) ... 2024-07-15T06:35:10.109 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up python3-ceph-common (17.2.7-1jammy) ... 2024-07-15T06:35:10.231 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-joblib (0.17.0-4ubuntu1) ... 2024-07-15T06:35:10.403 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up python3-idna (3.3-1ubuntu0.1) ... 2024-07-15T06:35:10.771 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up python3-sklearn (0.23.2-5ubuntu6) ... 2024-07-15T06:35:10.908 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-cachetools (5.0.0-1) ... 2024-07-15T06:35:11.184 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-openssl (21.0.0-1) ... 2024-07-15T06:35:11.468 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-bcrypt (3.2.0-1build1) ... 2024-07-15T06:35:11.736 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-pyinotify (0.9.6-1.3) ... 2024-07-15T06:35:11.935 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up libqt5core5a:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-07-15T06:35:11.995 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-threadpoolctl (3.1.0-1) ... 2024-07-15T06:35:12.086 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up python3-toml (0.10.2-1) ... 2024-07-15T06:35:12.247 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-ceph-argparse (17.2.7-1jammy) ... 2024-07-15T06:35:12.362 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up librdkafka1:amd64 (1.8.0-1build1) ... 2024-07-15T06:35:12.472 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up xmlstarlet (1.6.1-2.1) ... 2024-07-15T06:35:12.564 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-sklearn-lib:amd64 (0.23.2-5ubuntu6) ... 2024-07-15T06:35:12.573 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up python3-urllib3 (1.26.5-1~exp1ubuntu0.1) ... 2024-07-15T06:35:12.716 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-markupsafe (2.0.1-2build1) ... 2024-07-15T06:35:12.874 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up python3-pluggy (0.13.0-7.1) ... 2024-07-15T06:35:12.993 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up libpcre2-16-0:amd64 (10.39-3ubuntu0.1) ... 2024-07-15T06:35:13.120 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-psutil (5.9.0-1build1) ... 2024-07-15T06:35:13.133 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up python3-zc.lockfile (2.0-1) ... 2024-07-15T06:35:13.393 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up libqt5dbus5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-07-15T06:35:13.504 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python-babel-localedata (2.8.0+dfsg.1-7) ... 2024-07-15T06:35:13.536 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up libfmt8:amd64 (8.1.1+ds1-2) ... 2024-07-15T06:35:13.647 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-natsort (8.0.2-1) ... 2024-07-15T06:35:13.687 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up python3-pyasn1 (0.4.8-1) ... 2024-07-15T06:35:13.898 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-routes (2.5.1-1ubuntu1) ... 2024-07-15T06:35:14.014 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up python3-singledispatch (3.4.0.3-3) ... 2024-07-15T06:35:14.149 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-simplejson (3.17.6-1build1) ... 2024-07-15T06:35:14.281 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up python3-logutils (0.3.3-8) ... 2024-07-15T06:35:14.467 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-pygments (2.11.2+dfsg-2) ... 2024-07-15T06:35:14.541 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up ceph-mgr-modules-core (17.2.7-1jammy) ... 2024-07-15T06:35:14.667 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up python3-tempora (4.1.2-1) ... 2024-07-15T06:35:14.935 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up python3-simplegeneric (0.8.1-3) ... 2024-07-15T06:35:15.153 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-tempita (0.5.2-6ubuntu1) ... 2024-07-15T06:35:15.194 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up python3-prettytable (2.5.0-2) ... 2024-07-15T06:35:15.412 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python-pastedeploy-tpl (2.1.1-1) ... 2024-07-15T06:35:15.462 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up liblttng-ust1:amd64 (2.13.1-1ubuntu1) ... 2024-07-15T06:35:15.538 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up qttranslations5-l10n (5.15.3-1) ... 2024-07-15T06:35:15.613 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up python3-websocket (1.2.3-1) ... 2024-07-15T06:35:15.664 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-wcwidth (0.2.5+dfsg1-1) ... 2024-07-15T06:35:15.914 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up libonig5:amd64 (6.9.7.1-2build1) ... 2024-07-15T06:35:15.940 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-certifi (2020.6.20-1) ... 2024-07-15T06:35:16.041 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up python3-mako (1.1.3+ds1-2ubuntu0.1) ... 2024-07-15T06:35:16.208 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-paste (3.5.0+dfsg1-1) ... 2024-07-15T06:35:16.325 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up python3-webob (1:1.8.6-1.1) ... 2024-07-15T06:35:16.593 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-cheroot (8.5.2+ds1-1ubuntu3.1) ... 2024-07-15T06:35:16.626 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up python3-jaraco.collections (3.4.0-2) ... 2024-07-15T06:35:16.894 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up librados2 (17.2.7-1jammy) ... 2024-07-15T06:35:16.902 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-werkzeug (2.0.2+dfsg1-1ubuntu0.22.04.2) ... 2024-07-15T06:35:17.054 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up python3-babel (2.8.0+dfsg.1-7) ... 2024-07-15T06:35:17.262 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-jaraco.text (3.6.0-2) ... 2024-07-15T06:35:17.459 INFO:teuthology.orchestra.run.smithi134.stdout:update-alternatives: using /usr/bin/pybabel-python3 to provide /usr/bin/pybabel (pybabel) in auto mode 2024-07-15T06:35:17.505 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up socat (1.7.4.1-3ubuntu4) ... 2024-07-15T06:35:17.606 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-ceph-common (17.2.7-1jammy) ... 2024-07-15T06:35:17.715 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up libsqlite3-mod-ceph (17.2.7-1jammy) ... 2024-07-15T06:35:17.841 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up libjq1:amd64 (1.6-2.1ubuntu3) ... 2024-07-15T06:35:17.890 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-idna (3.3-1ubuntu0.1) ... 2024-07-15T06:35:17.967 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up python3-pytest (6.2.5-1ubuntu2) ... 2024-07-15T06:35:18.217 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-sklearn (0.23.2-5ubuntu6) ... 2024-07-15T06:35:18.368 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up python3-pastedeploy (2.1.1-1) ... 2024-07-15T06:35:18.636 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up python3-portend (3.0.0-1) ... 2024-07-15T06:35:18.888 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up libqt5network5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-07-15T06:35:18.997 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up libcephfs2 (17.2.7-1jammy) ... 2024-07-15T06:35:19.115 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up python3-jinja2 (3.0.3-1ubuntu0.2) ... 2024-07-15T06:35:19.339 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up libqt5core5a:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-07-15T06:35:19.433 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up libradosstriper1 (17.2.7-1jammy) ... 2024-07-15T06:35:19.465 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-toml (0.10.2-1) ... 2024-07-15T06:35:19.559 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up python3-pyasn1-modules (0.2.1-1) ... 2024-07-15T06:35:19.725 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up librdkafka1:amd64 (1.8.0-1build1) ... 2024-07-15T06:35:19.860 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up python3-requests (2.25.1+dfsg-2ubuntu0.1) ... 2024-07-15T06:35:19.875 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up xmlstarlet (1.6.1-2.1) ... 2024-07-15T06:35:20.027 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-urllib3 (1.26.5-1~exp1ubuntu0.1) ... 2024-07-15T06:35:20.136 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up librbd1 (17.2.7-1jammy) ... 2024-07-15T06:35:20.287 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up jq (1.6-2.1ubuntu3) ... 2024-07-15T06:35:20.336 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-pluggy (0.13.0-7.1) ... 2024-07-15T06:35:20.405 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up python3-rsa (4.8-1) ... 2024-07-15T06:35:20.588 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-zc.lockfile (2.0-1) ... 2024-07-15T06:35:20.656 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up python3-webtest (2.0.35-1) ... 2024-07-15T06:35:20.922 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up libqt5dbus5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-07-15T06:35:20.941 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up ceph-fuse (17.2.7-1jammy) ... 2024-07-15T06:35:21.023 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up libfmt8:amd64 (8.1.1+ds1-2) ... 2024-07-15T06:35:21.133 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-pyasn1 (0.4.8-1) ... 2024-07-15T06:35:21.154 INFO:teuthology.orchestra.run.smithi134.stdout:Created symlink /etc/systemd/system/remote-fs.target.wants/ceph-fuse.target -> /lib/systemd/system/ceph-fuse.target. 2024-07-15T06:35:21.154 INFO:teuthology.orchestra.run.smithi134.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-fuse.target -> /lib/systemd/system/ceph-fuse.target. 2024-07-15T06:35:21.434 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-singledispatch (3.4.0.3-3) ... 2024-07-15T06:35:21.692 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up libcephfs-dev (17.2.7-1jammy) ... 2024-07-15T06:35:21.702 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-logutils (0.3.3-8) ... 2024-07-15T06:35:21.961 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up ceph-mgr-modules-core (17.2.7-1jammy) ... 2024-07-15T06:35:22.071 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-tempora (4.1.2-1) ... 2024-07-15T06:35:22.151 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up python3-requests-oauthlib (1.3.0+ds-0.1) ... 2024-07-15T06:35:22.322 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-simplegeneric (0.8.1-3) ... 2024-07-15T06:35:22.511 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up python3-cherrypy3 (18.6.1-4) ... 2024-07-15T06:35:22.556 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-prettytable (2.5.0-2) ... 2024-07-15T06:35:22.816 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up liblttng-ust1:amd64 (2.13.1-1ubuntu1) ... 2024-07-15T06:35:22.905 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up python3-pastescript (2.0.2-4) ... 2024-07-15T06:35:22.942 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-websocket (1.2.3-1) ... 2024-07-15T06:35:23.181 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up python3-pecan (1.3.3-4ubuntu2) ... 2024-07-15T06:35:23.227 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up libonig5:amd64 (6.9.7.1-2build1) ... 2024-07-15T06:35:23.353 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-mako (1.1.3+ds1-2ubuntu0.1) ... 2024-07-15T06:35:23.610 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up libthrift-0.16.0:amd64 (0.16.0-2) ... 2024-07-15T06:35:23.645 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-webob (1:1.8.6-1.1) ... 2024-07-15T06:35:23.745 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up python3-rados (17.2.7-1jammy) ... 2024-07-15T06:35:23.863 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up librgw2 (17.2.7-1jammy) ... 2024-07-15T06:35:23.947 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-jaraco.collections (3.4.0-2) ... 2024-07-15T06:35:23.964 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up python3-rbd (17.2.7-1jammy) ... 2024-07-15T06:35:24.090 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up rbd-fuse (17.2.7-1jammy) ... 2024-07-15T06:35:24.181 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up librados2 (17.2.7-1jammy) ... 2024-07-15T06:35:24.216 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up python3-google-auth (1.5.1-3) ... 2024-07-15T06:35:24.299 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-babel (2.8.0+dfsg.1-7) ... 2024-07-15T06:35:24.492 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up python3-rgw (17.2.7-1jammy) ... 2024-07-15T06:35:24.637 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up python3-cephfs (17.2.7-1jammy) ... 2024-07-15T06:35:24.713 INFO:teuthology.orchestra.run.smithi089.stdout:update-alternatives: using /usr/bin/pybabel-python3 to provide /usr/bin/pybabel (pybabel) in auto mode 2024-07-15T06:35:24.787 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up python3-kubernetes (12.0.1-1ubuntu1) ... 2024-07-15T06:35:24.952 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up libsqlite3-mod-ceph (17.2.7-1jammy) ... 2024-07-15T06:35:25.078 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up libjq1:amd64 (1.6-2.1ubuntu3) ... 2024-07-15T06:35:25.229 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-pytest (6.2.5-1ubuntu2) ... 2024-07-15T06:35:25.605 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-pastedeploy (2.1.1-1) ... 2024-07-15T06:35:25.848 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-portend (3.0.0-1) ... 2024-07-15T06:35:26.057 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up ceph-common (17.2.7-1jammy) ... 2024-07-15T06:35:26.099 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up libqt5network5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-07-15T06:35:26.225 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up libcephfs2 (17.2.7-1jammy) ... 2024-07-15T06:35:26.351 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-jinja2 (3.0.3-1ubuntu0.2) ... 2024-07-15T06:35:26.659 INFO:teuthology.orchestra.run.smithi134.stdout:Adding group ceph....done 2024-07-15T06:35:26.669 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up libradosstriper1 (17.2.7-1jammy) ... 2024-07-15T06:35:26.798 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-pyasn1-modules (0.2.1-1) ... 2024-07-15T06:35:27.036 INFO:teuthology.orchestra.run.smithi134.stdout:Adding system user ceph....done 2024-07-15T06:35:27.096 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-requests (2.25.1+dfsg-2ubuntu0.1) ... 2024-07-15T06:35:27.213 INFO:teuthology.orchestra.run.smithi134.stdout:Setting system user ceph properties....done 2024-07-15T06:35:27.232 INFO:teuthology.orchestra.run.smithi134.stdout:chown: cannot access '/var/log/ceph/*.log*': No such file or directory 2024-07-15T06:35:27.339 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up librbd1 (17.2.7-1jammy) ... 2024-07-15T06:35:27.364 INFO:teuthology.orchestra.run.smithi134.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target -> /lib/systemd/system/ceph.target. 2024-07-15T06:35:27.440 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up jq (1.6-2.1ubuntu3) ... 2024-07-15T06:35:27.558 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-rsa (4.8-1) ... 2024-07-15T06:35:27.689 INFO:teuthology.orchestra.run.smithi134.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/rbdmap.service -> /lib/systemd/system/rbdmap.service. 2024-07-15T06:35:27.842 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-webtest (2.0.35-1) ... 2024-07-15T06:35:28.169 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up ceph-fuse (17.2.7-1jammy) ... 2024-07-15T06:35:28.192 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up ceph-test (17.2.7-1jammy) ... 2024-07-15T06:35:28.318 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up radosgw (17.2.7-1jammy) ... 2024-07-15T06:35:28.380 INFO:teuthology.orchestra.run.smithi089.stdout:Created symlink /etc/systemd/system/remote-fs.target.wants/ceph-fuse.target -> /lib/systemd/system/ceph-fuse.target. 2024-07-15T06:35:28.380 INFO:teuthology.orchestra.run.smithi089.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-fuse.target -> /lib/systemd/system/ceph-fuse.target. 2024-07-15T06:35:28.596 INFO:teuthology.orchestra.run.smithi134.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target -> /lib/systemd/system/ceph-radosgw.target. 2024-07-15T06:35:28.596 INFO:teuthology.orchestra.run.smithi134.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target -> /lib/systemd/system/ceph-radosgw.target. 2024-07-15T06:35:28.870 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up libcephfs-dev (17.2.7-1jammy) ... 2024-07-15T06:35:28.979 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-requests-oauthlib (1.3.0+ds-0.1) ... 2024-07-15T06:35:29.230 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-cherrypy3 (18.6.1-4) ... 2024-07-15T06:35:29.304 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up ceph-base (17.2.7-1jammy) ... 2024-07-15T06:35:29.598 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-pastescript (2.0.2-4) ... 2024-07-15T06:35:29.603 INFO:teuthology.orchestra.run.smithi134.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service -> /lib/systemd/system/ceph-crash.service. 2024-07-15T06:35:29.883 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-pecan (1.3.3-4ubuntu2) ... 2024-07-15T06:35:30.056 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up ceph-mds (17.2.7-1jammy) ... 2024-07-15T06:35:30.281 INFO:teuthology.orchestra.run.smithi134.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target -> /lib/systemd/system/ceph-mds.target. 2024-07-15T06:35:30.281 INFO:teuthology.orchestra.run.smithi134.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target -> /lib/systemd/system/ceph-mds.target. 2024-07-15T06:35:30.338 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up libthrift-0.16.0:amd64 (0.16.0-2) ... 2024-07-15T06:35:30.464 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-rados (17.2.7-1jammy) ... 2024-07-15T06:35:30.565 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up librgw2 (17.2.7-1jammy) ... 2024-07-15T06:35:30.691 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-rbd (17.2.7-1jammy) ... 2024-07-15T06:35:30.774 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up ceph-mgr (17.2.7-1jammy) ... 2024-07-15T06:35:30.817 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up rbd-fuse (17.2.7-1jammy) ... 2024-07-15T06:35:30.943 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-google-auth (1.5.1-3) ... 2024-07-15T06:35:30.984 INFO:teuthology.orchestra.run.smithi134.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target -> /lib/systemd/system/ceph-mgr.target. 2024-07-15T06:35:30.984 INFO:teuthology.orchestra.run.smithi134.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target -> /lib/systemd/system/ceph-mgr.target. 2024-07-15T06:35:31.236 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-rgw (17.2.7-1jammy) ... 2024-07-15T06:35:31.362 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-cephfs (17.2.7-1jammy) ... 2024-07-15T06:35:31.459 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up ceph-osd (17.2.7-1jammy) ... 2024-07-15T06:35:31.488 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-kubernetes (12.0.1-1ubuntu1) ... 2024-07-15T06:35:31.729 INFO:teuthology.orchestra.run.smithi134.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target -> /lib/systemd/system/ceph-osd.target. 2024-07-15T06:35:31.729 INFO:teuthology.orchestra.run.smithi134.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target -> /lib/systemd/system/ceph-osd.target. 2024-07-15T06:35:32.211 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up ceph-mgr-k8sevents (17.2.7-1jammy) ... 2024-07-15T06:35:32.337 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up ceph-mgr-diskprediction-local (17.2.7-1jammy) ... 2024-07-15T06:35:32.471 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up ceph-mon (17.2.7-1jammy) ... 2024-07-15T06:35:32.681 INFO:teuthology.orchestra.run.smithi134.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target -> /lib/systemd/system/ceph-mon.target. 2024-07-15T06:35:32.681 INFO:teuthology.orchestra.run.smithi134.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target -> /lib/systemd/system/ceph-mon.target. 2024-07-15T06:35:32.742 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up ceph-common (17.2.7-1jammy) ... 2024-07-15T06:35:33.156 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up ceph-mgr-cephadm (17.2.7-1jammy) ... 2024-07-15T06:35:33.252 INFO:teuthology.orchestra.run.smithi089.stdout:Adding group ceph....done 2024-07-15T06:35:33.299 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up ceph (17.2.7-1jammy) ... 2024-07-15T06:35:33.425 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up ceph-mgr-dashboard (17.2.7-1jammy) ... 2024-07-15T06:35:33.593 INFO:teuthology.orchestra.run.smithi134.stdout:Setting up ceph-volume (17.2.7-1jammy) ... 2024-07-15T06:35:33.704 INFO:teuthology.orchestra.run.smithi089.stdout:Adding system user ceph....done 2024-07-15T06:35:33.720 INFO:teuthology.orchestra.run.smithi134.stdout:Processing triggers for man-db (2.10.2-1) ... 2024-07-15T06:35:33.881 INFO:teuthology.orchestra.run.smithi089.stdout:Setting system user ceph properties....done 2024-07-15T06:35:33.896 INFO:teuthology.orchestra.run.smithi089.stdout:chown: cannot access '/var/log/ceph/*.log*': No such file or directory 2024-07-15T06:35:34.026 INFO:teuthology.orchestra.run.smithi089.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target -> /lib/systemd/system/ceph.target. 2024-07-15T06:35:34.335 INFO:teuthology.orchestra.run.smithi089.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/rbdmap.service -> /lib/systemd/system/rbdmap.service. 2024-07-15T06:35:34.836 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up ceph-test (17.2.7-1jammy) ... 2024-07-15T06:35:34.938 INFO:teuthology.orchestra.run.smithi134.stdout:Processing triggers for libc-bin (2.35-0ubuntu3.1) ... 2024-07-15T06:35:34.962 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up radosgw (17.2.7-1jammy) ... 2024-07-15T06:35:35.224 INFO:teuthology.orchestra.run.smithi089.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target -> /lib/systemd/system/ceph-radosgw.target. 2024-07-15T06:35:35.224 INFO:teuthology.orchestra.run.smithi089.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target -> /lib/systemd/system/ceph-radosgw.target. 2024-07-15T06:35:35.404 INFO:teuthology.orchestra.run.smithi134.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2024-07-15T06:35:35.409 DEBUG:teuthology.parallel:result is None 2024-07-15T06:35:35.906 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up ceph-base (17.2.7-1jammy) ... 2024-07-15T06:35:36.233 INFO:teuthology.orchestra.run.smithi089.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service -> /lib/systemd/system/ceph-crash.service. 2024-07-15T06:35:36.732 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up ceph-mds (17.2.7-1jammy) ... 2024-07-15T06:35:36.945 INFO:teuthology.orchestra.run.smithi089.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target -> /lib/systemd/system/ceph-mds.target. 2024-07-15T06:35:36.945 INFO:teuthology.orchestra.run.smithi089.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target -> /lib/systemd/system/ceph-mds.target. 2024-07-15T06:35:37.467 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up ceph-mgr (17.2.7-1jammy) ... 2024-07-15T06:35:37.685 INFO:teuthology.orchestra.run.smithi089.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target -> /lib/systemd/system/ceph-mgr.target. 2024-07-15T06:35:37.685 INFO:teuthology.orchestra.run.smithi089.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target -> /lib/systemd/system/ceph-mgr.target. 2024-07-15T06:35:38.185 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up ceph-osd (17.2.7-1jammy) ... 2024-07-15T06:35:38.471 INFO:teuthology.orchestra.run.smithi089.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target -> /lib/systemd/system/ceph-osd.target. 2024-07-15T06:35:38.471 INFO:teuthology.orchestra.run.smithi089.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target -> /lib/systemd/system/ceph-osd.target. 2024-07-15T06:35:38.929 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up ceph-mgr-k8sevents (17.2.7-1jammy) ... 2024-07-15T06:35:39.180 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up ceph-mgr-diskprediction-local (17.2.7-1jammy) ... 2024-07-15T06:35:39.314 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up ceph-mon (17.2.7-1jammy) ... 2024-07-15T06:35:39.516 INFO:teuthology.orchestra.run.smithi089.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target -> /lib/systemd/system/ceph-mon.target. 2024-07-15T06:35:39.516 INFO:teuthology.orchestra.run.smithi089.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target -> /lib/systemd/system/ceph-mon.target. 2024-07-15T06:35:40.007 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up ceph-mgr-cephadm (17.2.7-1jammy) ... 2024-07-15T06:35:40.158 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up ceph (17.2.7-1jammy) ... 2024-07-15T06:35:40.285 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up ceph-mgr-dashboard (17.2.7-1jammy) ... 2024-07-15T06:35:40.436 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up ceph-volume (17.2.7-1jammy) ... 2024-07-15T06:35:40.563 INFO:teuthology.orchestra.run.smithi089.stdout:Processing triggers for man-db (2.10.2-1) ... 2024-07-15T06:35:41.690 INFO:teuthology.orchestra.run.smithi089.stdout:Processing triggers for libc-bin (2.35-0ubuntu3.1) ... 2024-07-15T06:35:42.159 INFO:teuthology.orchestra.run.smithi089.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2024-07-15T06:35:42.164 DEBUG:teuthology.parallel:result is None 2024-07-15T06:35:42.165 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-07-15T06:35:42.165 INFO:teuthology.packaging:ref: None 2024-07-15T06:35:42.165 INFO:teuthology.packaging:tag: None 2024-07-15T06:35:42.165 INFO:teuthology.packaging:branch: quincy 2024-07-15T06:35:42.165 INFO:teuthology.packaging:sha1: 6ea7aaa9f93d34dc4d21862afe00140dfddbd706 2024-07-15T06:35:42.165 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&ref=quincy 2024-07-15T06:35:42.313 DEBUG:teuthology.orchestra.run.smithi089:> dpkg-query -W -f '${Version}' ceph 2024-07-15T06:35:42.331 INFO:teuthology.orchestra.run.smithi089.stdout:17.2.7-1jammy 2024-07-15T06:35:42.332 INFO:teuthology.packaging:The installed version of ceph is 17.2.7-1jammy 2024-07-15T06:35:42.332 INFO:teuthology.task.install:The correct ceph version 17.2.7-1jammy is installed. 2024-07-15T06:35:42.333 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-07-15T06:35:42.333 INFO:teuthology.packaging:ref: None 2024-07-15T06:35:42.334 INFO:teuthology.packaging:tag: None 2024-07-15T06:35:42.334 INFO:teuthology.packaging:branch: quincy 2024-07-15T06:35:42.334 INFO:teuthology.packaging:sha1: 6ea7aaa9f93d34dc4d21862afe00140dfddbd706 2024-07-15T06:35:42.334 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&ref=quincy 2024-07-15T06:35:42.472 DEBUG:teuthology.orchestra.run.smithi134:> dpkg-query -W -f '${Version}' ceph 2024-07-15T06:35:42.490 INFO:teuthology.orchestra.run.smithi134.stdout:17.2.7-1jammy 2024-07-15T06:35:42.491 INFO:teuthology.packaging:The installed version of ceph is 17.2.7-1jammy 2024-07-15T06:35:42.491 INFO:teuthology.task.install:The correct ceph version 17.2.7-1jammy is installed. 2024-07-15T06:35:42.492 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2024-07-15T06:35:42.493 DEBUG:teuthology.orchestra.run.smithi089:> set -ex 2024-07-15T06:35:42.493 DEBUG:teuthology.orchestra.run.smithi089:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-07-15T06:35:42.506 DEBUG:teuthology.orchestra.run.smithi134:> set -ex 2024-07-15T06:35:42.506 DEBUG:teuthology.orchestra.run.smithi134:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-07-15T06:35:42.544 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2024-07-15T06:35:42.544 DEBUG:teuthology.orchestra.run.smithi089:> set -ex 2024-07-15T06:35:42.544 DEBUG:teuthology.orchestra.run.smithi089:> sudo dd of=/usr/bin/daemon-helper 2024-07-15T06:35:42.561 DEBUG:teuthology.orchestra.run.smithi089:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-07-15T06:35:42.617 DEBUG:teuthology.orchestra.run.smithi134:> set -ex 2024-07-15T06:35:42.617 DEBUG:teuthology.orchestra.run.smithi134:> sudo dd of=/usr/bin/daemon-helper 2024-07-15T06:35:42.630 DEBUG:teuthology.orchestra.run.smithi134:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-07-15T06:35:42.684 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2024-07-15T06:35:42.684 DEBUG:teuthology.orchestra.run.smithi089:> set -ex 2024-07-15T06:35:42.684 DEBUG:teuthology.orchestra.run.smithi089:> sudo dd of=/usr/bin/adjust-ulimits 2024-07-15T06:35:42.697 DEBUG:teuthology.orchestra.run.smithi089:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-07-15T06:35:42.752 DEBUG:teuthology.orchestra.run.smithi134:> set -ex 2024-07-15T06:35:42.752 DEBUG:teuthology.orchestra.run.smithi134:> sudo dd of=/usr/bin/adjust-ulimits 2024-07-15T06:35:42.767 DEBUG:teuthology.orchestra.run.smithi134:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-07-15T06:35:42.823 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2024-07-15T06:35:42.823 DEBUG:teuthology.orchestra.run.smithi089:> set -ex 2024-07-15T06:35:42.824 DEBUG:teuthology.orchestra.run.smithi089:> sudo dd of=/usr/bin/stdin-killer 2024-07-15T06:35:42.836 DEBUG:teuthology.orchestra.run.smithi089:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-07-15T06:35:42.888 DEBUG:teuthology.orchestra.run.smithi134:> set -ex 2024-07-15T06:35:42.888 DEBUG:teuthology.orchestra.run.smithi134:> sudo dd of=/usr/bin/stdin-killer 2024-07-15T06:35:42.902 DEBUG:teuthology.orchestra.run.smithi134:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-07-15T06:35:42.955 INFO:teuthology.run_tasks:Running task cephadm... 2024-07-15T06:35:43.053 INFO:tasks.cephadm:Config: {'cephadm_branch': 'quincy', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'conf': {'osd': {'osd_class_default_list': '*', 'osd_class_load_list': '*', 'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}, 'global': {'mon election default strategy': 1}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}}, 'image': 'quay.ceph.io/ceph-ci/ceph:quincy', 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': '6ea7aaa9f93d34dc4d21862afe00140dfddbd706'} 2024-07-15T06:35:43.053 INFO:tasks.cephadm:Cluster image is quay.ceph.io/ceph-ci/ceph:quincy 2024-07-15T06:35:43.053 INFO:tasks.cephadm:Cluster fsid is 75ba02bc-4274-11ef-bcad-c7b262605968 2024-07-15T06:35:43.054 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-07-15T06:35:43.054 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.89', 'mon.c': '[v2:172.21.15.89:3301,v1:172.21.15.89:6790]', 'mon.b': '172.21.15.134'} 2024-07-15T06:35:43.054 INFO:tasks.cephadm:First mon is mon.a on smithi089 2024-07-15T06:35:43.054 INFO:tasks.cephadm:First mgr is y 2024-07-15T06:35:43.054 INFO:tasks.cephadm:Normalizing hostnames... 2024-07-15T06:35:43.054 DEBUG:teuthology.orchestra.run.smithi089:> sudo hostname $(hostname -s) 2024-07-15T06:35:43.069 DEBUG:teuthology.orchestra.run.smithi134:> sudo hostname $(hostname -s) 2024-07-15T06:35:43.084 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref quincy)... 2024-07-15T06:35:43.084 DEBUG:teuthology.orchestra.run.smithi089:> curl --silent https://raw.githubusercontent.com/ceph/ceph/quincy/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-07-15T06:35:43.338 INFO:teuthology.orchestra.run.smithi089.stdout:-rw-rw-r-- 1 ubuntu ubuntu 375153 Jul 15 06:35 /home/ubuntu/cephtest/cephadm 2024-07-15T06:35:43.339 DEBUG:teuthology.orchestra.run.smithi134:> curl --silent https://raw.githubusercontent.com/ceph/ceph/quincy/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-07-15T06:35:43.561 INFO:teuthology.orchestra.run.smithi134.stdout:-rw-rw-r-- 1 ubuntu ubuntu 375153 Jul 15 06:35 /home/ubuntu/cephtest/cephadm 2024-07-15T06:35:43.561 DEBUG:teuthology.orchestra.run.smithi089:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-07-15T06:35:43.570 DEBUG:teuthology.orchestra.run.smithi134:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-07-15T06:35:43.587 INFO:tasks.cephadm:Pulling image quay.ceph.io/ceph-ci/ceph:quincy on all hosts... 2024-07-15T06:35:43.587 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy pull 2024-07-15T06:35:43.614 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy pull 2024-07-15T06:35:43.781 INFO:teuthology.orchestra.run.smithi089.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2024-07-15T06:35:43.789 INFO:teuthology.orchestra.run.smithi134.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2024-07-15T06:36:54.543 INFO:teuthology.orchestra.run.smithi089.stdout:{ 2024-07-15T06:36:54.543 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph_version": "ceph version 17.2.7-1102-g9a202024 (9a2020246f85463d4fdb115c94aa40b463caa1d2) quincy (stable)", 2024-07-15T06:36:54.544 INFO:teuthology.orchestra.run.smithi089.stdout: "image_id": "280b03ed686956e54e9ec4635099bb3f3277eb92f55632a07022dbb84b49fcef", 2024-07-15T06:36:54.544 INFO:teuthology.orchestra.run.smithi089.stdout: "repo_digests": [ 2024-07-15T06:36:54.544 INFO:teuthology.orchestra.run.smithi089.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:4bff20d5b645298ccfe006b4cead5811c6d6f9d410f45254b24b3d1d71199b25" 2024-07-15T06:36:54.544 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2024-07-15T06:36:54.544 INFO:teuthology.orchestra.run.smithi089.stdout:} 2024-07-15T06:36:56.122 INFO:teuthology.orchestra.run.smithi134.stdout:{ 2024-07-15T06:36:56.123 INFO:teuthology.orchestra.run.smithi134.stdout: "ceph_version": "ceph version 17.2.7-1102-g9a202024 (9a2020246f85463d4fdb115c94aa40b463caa1d2) quincy (stable)", 2024-07-15T06:36:56.123 INFO:teuthology.orchestra.run.smithi134.stdout: "image_id": "280b03ed686956e54e9ec4635099bb3f3277eb92f55632a07022dbb84b49fcef", 2024-07-15T06:36:56.123 INFO:teuthology.orchestra.run.smithi134.stdout: "repo_digests": [ 2024-07-15T06:36:56.123 INFO:teuthology.orchestra.run.smithi134.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:4bff20d5b645298ccfe006b4cead5811c6d6f9d410f45254b24b3d1d71199b25" 2024-07-15T06:36:56.123 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-07-15T06:36:56.123 INFO:teuthology.orchestra.run.smithi134.stdout:} 2024-07-15T06:36:56.162 DEBUG:teuthology.orchestra.run.smithi089:> sudo mkdir -p /etc/ceph 2024-07-15T06:36:56.177 DEBUG:teuthology.orchestra.run.smithi134:> sudo mkdir -p /etc/ceph 2024-07-15T06:36:56.191 DEBUG:teuthology.orchestra.run.smithi089:> sudo chmod 777 /etc/ceph 2024-07-15T06:36:56.232 DEBUG:teuthology.orchestra.run.smithi134:> sudo chmod 777 /etc/ceph 2024-07-15T06:36:56.246 INFO:tasks.cephadm:Writing seed config... 2024-07-15T06:36:56.247 INFO:tasks.cephadm: override: [osd] osd_class_default_list = * 2024-07-15T06:36:56.247 INFO:tasks.cephadm: override: [osd] osd_class_load_list = * 2024-07-15T06:36:56.247 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-07-15T06:36:56.247 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-07-15T06:36:56.248 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2024-07-15T06:36:56.248 INFO:tasks.cephadm: override: [global] mon election default strategy = 1 2024-07-15T06:36:56.248 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-07-15T06:36:56.248 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-07-15T06:36:56.248 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-07-15T06:36:56.248 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-07-15T06:36:56.248 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-07-15T06:36:56.249 DEBUG:teuthology.orchestra.run.smithi089:> set -ex 2024-07-15T06:36:56.249 DEBUG:teuthology.orchestra.run.smithi089:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-07-15T06:36:56.280 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = 75ba02bc-4274-11ef-bcad-c7b262605968 mon election default strategy = 1 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true osd_class_default_list = * osd_class_load_list = * debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-07-15T06:36:56.281 DEBUG:teuthology.orchestra.run.smithi089:mon.a> sudo journalctl -f -n 0 -u ceph-75ba02bc-4274-11ef-bcad-c7b262605968@mon.a.service 2024-07-15T06:36:56.326 DEBUG:teuthology.orchestra.run.smithi089:mgr.y> sudo journalctl -f -n 0 -u ceph-75ba02bc-4274-11ef-bcad-c7b262605968@mgr.y.service 2024-07-15T06:36:56.369 INFO:tasks.cephadm:Bootstrapping... 2024-07-15T06:36:56.369 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy -v bootstrap --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.89 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-07-15T06:36:56.580 INFO:teuthology.orchestra.run.smithi089.stdout:-------------------------------------------------------------------------------- 2024-07-15T06:36:56.580 INFO:teuthology.orchestra.run.smithi089.stdout:cephadm ['--image', 'quay.ceph.io/ceph-ci/ceph:quincy', '-v', 'bootstrap', '--fsid', '75ba02bc-4274-11ef-bcad-c7b262605968', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'y', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.89', '--skip-admin-label'] 2024-07-15T06:36:56.580 INFO:teuthology.orchestra.run.smithi089.stderr:Specifying an fsid for your cluster offers no advantages and may increase the likelihood of fsid conflicts. 2024-07-15T06:36:56.581 INFO:teuthology.orchestra.run.smithi089.stdout:Verifying podman|docker is present... 2024-07-15T06:36:56.581 INFO:teuthology.orchestra.run.smithi089.stdout:Verifying lvm2 is present... 2024-07-15T06:36:56.581 INFO:teuthology.orchestra.run.smithi089.stdout:Verifying time synchronization is in place... 2024-07-15T06:36:56.586 INFO:teuthology.orchestra.run.smithi089.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2024-07-15T06:36:56.586 INFO:teuthology.orchestra.run.smithi089.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2024-07-15T06:36:56.589 INFO:teuthology.orchestra.run.smithi089.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2024-07-15T06:36:56.589 INFO:teuthology.orchestra.run.smithi089.stdout:systemctl: stdout inactive 2024-07-15T06:36:56.593 INFO:teuthology.orchestra.run.smithi089.stdout:Non-zero exit code 1 from systemctl is-enabled chronyd.service 2024-07-15T06:36:56.593 INFO:teuthology.orchestra.run.smithi089.stdout:systemctl: stderr Failed to get unit file state for chronyd.service: No such file or directory 2024-07-15T06:36:56.597 INFO:teuthology.orchestra.run.smithi089.stdout:Non-zero exit code 3 from systemctl is-active chronyd.service 2024-07-15T06:36:56.597 INFO:teuthology.orchestra.run.smithi089.stdout:systemctl: stdout inactive 2024-07-15T06:36:56.600 INFO:teuthology.orchestra.run.smithi089.stdout:Non-zero exit code 1 from systemctl is-enabled systemd-timesyncd.service 2024-07-15T06:36:56.601 INFO:teuthology.orchestra.run.smithi089.stdout:systemctl: stdout masked 2024-07-15T06:36:56.604 INFO:teuthology.orchestra.run.smithi089.stdout:Non-zero exit code 3 from systemctl is-active systemd-timesyncd.service 2024-07-15T06:36:56.604 INFO:teuthology.orchestra.run.smithi089.stdout:systemctl: stdout inactive 2024-07-15T06:36:56.607 INFO:teuthology.orchestra.run.smithi089.stdout:Non-zero exit code 1 from systemctl is-enabled ntpd.service 2024-07-15T06:36:56.607 INFO:teuthology.orchestra.run.smithi089.stdout:systemctl: stderr Failed to get unit file state for ntpd.service: No such file or directory 2024-07-15T06:36:56.611 INFO:teuthology.orchestra.run.smithi089.stdout:Non-zero exit code 3 from systemctl is-active ntpd.service 2024-07-15T06:36:56.611 INFO:teuthology.orchestra.run.smithi089.stdout:systemctl: stdout inactive 2024-07-15T06:36:56.614 INFO:teuthology.orchestra.run.smithi089.stdout:systemctl: stdout enabled 2024-07-15T06:36:56.618 INFO:teuthology.orchestra.run.smithi089.stdout:systemctl: stdout active 2024-07-15T06:36:56.618 INFO:teuthology.orchestra.run.smithi089.stdout:Unit ntp.service is enabled and running 2024-07-15T06:36:56.618 INFO:teuthology.orchestra.run.smithi089.stdout:Repeating the final host check... 2024-07-15T06:36:56.618 INFO:teuthology.orchestra.run.smithi089.stdout:docker (/usr/bin/docker) is present 2024-07-15T06:36:56.618 INFO:teuthology.orchestra.run.smithi089.stdout:systemctl is present 2024-07-15T06:36:56.618 INFO:teuthology.orchestra.run.smithi089.stdout:lvcreate is present 2024-07-15T06:36:56.621 INFO:teuthology.orchestra.run.smithi089.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2024-07-15T06:36:56.621 INFO:teuthology.orchestra.run.smithi089.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2024-07-15T06:36:56.624 INFO:teuthology.orchestra.run.smithi089.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2024-07-15T06:36:56.624 INFO:teuthology.orchestra.run.smithi089.stdout:systemctl: stdout inactive 2024-07-15T06:36:56.627 INFO:teuthology.orchestra.run.smithi089.stdout:Non-zero exit code 1 from systemctl is-enabled chronyd.service 2024-07-15T06:36:56.627 INFO:teuthology.orchestra.run.smithi089.stdout:systemctl: stderr Failed to get unit file state for chronyd.service: No such file or directory 2024-07-15T06:36:56.631 INFO:teuthology.orchestra.run.smithi089.stdout:Non-zero exit code 3 from systemctl is-active chronyd.service 2024-07-15T06:36:56.631 INFO:teuthology.orchestra.run.smithi089.stdout:systemctl: stdout inactive 2024-07-15T06:36:56.634 INFO:teuthology.orchestra.run.smithi089.stdout:Non-zero exit code 1 from systemctl is-enabled systemd-timesyncd.service 2024-07-15T06:36:56.634 INFO:teuthology.orchestra.run.smithi089.stdout:systemctl: stdout masked 2024-07-15T06:36:56.637 INFO:teuthology.orchestra.run.smithi089.stdout:Non-zero exit code 3 from systemctl is-active systemd-timesyncd.service 2024-07-15T06:36:56.637 INFO:teuthology.orchestra.run.smithi089.stdout:systemctl: stdout inactive 2024-07-15T06:36:56.642 INFO:teuthology.orchestra.run.smithi089.stdout:Non-zero exit code 1 from systemctl is-enabled ntpd.service 2024-07-15T06:36:56.642 INFO:teuthology.orchestra.run.smithi089.stdout:systemctl: stderr Failed to get unit file state for ntpd.service: No such file or directory 2024-07-15T06:36:56.644 INFO:teuthology.orchestra.run.smithi089.stdout:Non-zero exit code 3 from systemctl is-active ntpd.service 2024-07-15T06:36:56.644 INFO:teuthology.orchestra.run.smithi089.stdout:systemctl: stdout inactive 2024-07-15T06:36:56.649 INFO:teuthology.orchestra.run.smithi089.stdout:systemctl: stdout enabled 2024-07-15T06:36:56.653 INFO:teuthology.orchestra.run.smithi089.stdout:systemctl: stdout active 2024-07-15T06:36:56.654 INFO:teuthology.orchestra.run.smithi089.stdout:Unit ntp.service is enabled and running 2024-07-15T06:36:56.654 INFO:teuthology.orchestra.run.smithi089.stdout:Host looks OK 2024-07-15T06:36:56.654 INFO:teuthology.orchestra.run.smithi089.stdout:Cluster fsid: 75ba02bc-4274-11ef-bcad-c7b262605968 2024-07-15T06:36:56.654 INFO:teuthology.orchestra.run.smithi089.stdout:Acquiring lock 140687547737776 on /run/cephadm/75ba02bc-4274-11ef-bcad-c7b262605968.lock 2024-07-15T06:36:56.654 INFO:teuthology.orchestra.run.smithi089.stdout:Lock 140687547737776 acquired on /run/cephadm/75ba02bc-4274-11ef-bcad-c7b262605968.lock 2024-07-15T06:36:56.654 INFO:teuthology.orchestra.run.smithi089.stdout:Verifying IP 172.21.15.89 port 3300 ... 2024-07-15T06:36:56.654 INFO:teuthology.orchestra.run.smithi089.stdout:Verifying IP 172.21.15.89 port 6789 ... 2024-07-15T06:36:56.655 INFO:teuthology.orchestra.run.smithi089.stdout:Base mon IP(s) is [172.21.15.89:3300, 172.21.15.89:6789], mon addrv is [v2:172.21.15.89:3300,v1:172.21.15.89:6789] 2024-07-15T06:36:56.657 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/sbin/ip: stdout default via 172.21.15.254 dev enp3s0f1 2024-07-15T06:36:56.658 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/sbin/ip: stdout default via 172.21.15.254 dev enp3s0f1 proto dhcp src 172.21.15.89 metric 100 2024-07-15T06:36:56.658 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/sbin/ip: stdout 158.69.67.47 via 172.21.15.254 dev enp3s0f1 proto dhcp src 172.21.15.89 metric 100 2024-07-15T06:36:56.658 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/sbin/ip: stdout 172.17.0.0/16 dev docker0 proto kernel scope link src 172.17.0.1 linkdown 2024-07-15T06:36:56.658 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/sbin/ip: stdout 172.21.0.0/20 dev enp3s0f1 proto kernel scope link src 172.21.15.89 metric 100 2024-07-15T06:36:56.658 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/sbin/ip: stdout 172.21.0.1 dev enp3s0f1 proto dhcp scope link src 172.21.15.89 metric 100 2024-07-15T06:36:56.658 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/sbin/ip: stdout 172.21.0.2 dev enp3s0f1 proto dhcp scope link src 172.21.15.89 metric 100 2024-07-15T06:36:56.658 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/sbin/ip: stdout 172.21.15.254 dev enp3s0f1 proto dhcp scope link src 172.21.15.89 metric 100 2024-07-15T06:36:56.659 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/sbin/ip: stdout ::1 dev lo proto kernel metric 256 pref medium 2024-07-15T06:36:56.660 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/sbin/ip: stdout fe80::/64 dev enp3s0f1 proto kernel metric 256 pref medium 2024-07-15T06:36:56.660 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/sbin/ip: stdout default via fe80::327c:5e00:6487:71e0 dev enp3s0f1 proto ra metric 100 expires 1785sec pref medium 2024-07-15T06:36:56.661 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/sbin/ip: stdout 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-07-15T06:36:56.661 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/sbin/ip: stdout inet6 ::1/128 scope host 2024-07-15T06:36:56.662 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/sbin/ip: stdout valid_lft forever preferred_lft forever 2024-07-15T06:36:56.662 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/sbin/ip: stdout 5: enp3s0f1: mtu 1500 state UP qlen 1000 2024-07-15T06:36:56.662 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/sbin/ip: stdout inet6 fe80::ec4:7aff:fe8f:d501/64 scope link 2024-07-15T06:36:56.662 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/sbin/ip: stdout valid_lft forever preferred_lft forever 2024-07-15T06:36:56.663 INFO:teuthology.orchestra.run.smithi089.stdout:Mon IP `172.21.15.89` is in CIDR network `172.21.0.0/20` 2024-07-15T06:36:56.663 INFO:teuthology.orchestra.run.smithi089.stdout:Mon IP `172.21.15.89` is in CIDR network `172.21.0.0/20` 2024-07-15T06:36:56.663 INFO:teuthology.orchestra.run.smithi089.stdout:Mon IP `172.21.15.89` is in CIDR network `172.21.0.1/32` 2024-07-15T06:36:56.663 INFO:teuthology.orchestra.run.smithi089.stdout:Mon IP `172.21.15.89` is in CIDR network `172.21.0.1/32` 2024-07-15T06:36:56.663 INFO:teuthology.orchestra.run.smithi089.stdout:Mon IP `172.21.15.89` is in CIDR network `172.21.0.2/32` 2024-07-15T06:36:56.663 INFO:teuthology.orchestra.run.smithi089.stdout:Mon IP `172.21.15.89` is in CIDR network `172.21.0.2/32` 2024-07-15T06:36:56.663 INFO:teuthology.orchestra.run.smithi089.stdout:Mon IP `172.21.15.89` is in CIDR network `172.21.15.254/32` 2024-07-15T06:36:56.663 INFO:teuthology.orchestra.run.smithi089.stdout:Mon IP `172.21.15.89` is in CIDR network `172.21.15.254/32` 2024-07-15T06:36:56.663 INFO:teuthology.orchestra.run.smithi089.stdout:Inferred mon public CIDR from local network configuration ['172.21.0.0/20', '172.21.0.0/20', '172.21.0.1/32', '172.21.0.1/32', '172.21.0.2/32', '172.21.0.2/32', '172.21.15.254/32', '172.21.15.254/32'] 2024-07-15T06:36:56.664 INFO:teuthology.orchestra.run.smithi089.stdout:Internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-07-15T06:36:56.664 INFO:teuthology.orchestra.run.smithi089.stdout:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2024-07-15T06:36:56.952 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/docker: stdout quincy: Pulling from ceph-ci/ceph 2024-07-15T06:36:56.953 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/docker: stdout Digest: sha256:4bff20d5b645298ccfe006b4cead5811c6d6f9d410f45254b24b3d1d71199b25 2024-07-15T06:36:56.953 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/docker: stdout Status: Image is up to date for quay.ceph.io/ceph-ci/ceph:quincy 2024-07-15T06:36:56.953 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/docker: stdout quay.ceph.io/ceph-ci/ceph:quincy 2024-07-15T06:36:58.527 INFO:teuthology.orchestra.run.smithi089.stdout:ceph: stdout ceph version 17.2.7-1102-g9a202024 (9a2020246f85463d4fdb115c94aa40b463caa1d2) quincy (stable) 2024-07-15T06:36:58.527 INFO:teuthology.orchestra.run.smithi089.stdout:Ceph version: ceph version 17.2.7-1102-g9a202024 (9a2020246f85463d4fdb115c94aa40b463caa1d2) quincy (stable) 2024-07-15T06:36:58.527 INFO:teuthology.orchestra.run.smithi089.stdout:Extracting ceph user uid/gid from container image... 2024-07-15T06:36:59.917 INFO:teuthology.orchestra.run.smithi089.stdout:stat: stdout 167 167 2024-07-15T06:36:59.918 INFO:teuthology.orchestra.run.smithi089.stdout:Creating initial keys... 2024-07-15T06:37:01.288 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-authtool: stdout AQCMw5RmqCBnLRAAkavUkaGV+UNv+62n33FYNw== 2024-07-15T06:37:02.701 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-authtool: stdout AQCOw5RmIr3lCRAAa4/gNbWDASos1nwrORQlTg== 2024-07-15T06:37:04.050 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-authtool: stdout AQCPw5RmV3H0IRAAE0V1gPuQ5xzgcz34p0JcUA== 2024-07-15T06:37:04.051 INFO:teuthology.orchestra.run.smithi089.stdout:Creating initial monmap... 2024-07-15T06:37:05.565 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: monmap file /tmp/monmap 2024-07-15T06:37:05.565 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/monmaptool: stdout setting min_mon_release = octopus 2024-07-15T06:37:05.565 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: set fsid to 75ba02bc-4274-11ef-bcad-c7b262605968 2024-07-15T06:37:05.565 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-07-15T06:37:05.565 INFO:teuthology.orchestra.run.smithi089.stdout:monmaptool for a [v2:172.21.15.89:3300,v1:172.21.15.89:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-07-15T06:37:05.565 INFO:teuthology.orchestra.run.smithi089.stdout:setting min_mon_release = octopus 2024-07-15T06:37:05.565 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/monmaptool: set fsid to 75ba02bc-4274-11ef-bcad-c7b262605968 2024-07-15T06:37:05.566 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-07-15T06:37:05.566 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-07-15T06:37:05.566 INFO:teuthology.orchestra.run.smithi089.stdout:Creating mon... 2024-07-15T06:37:06.989 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 0 set uid:gid to 167:167 (ceph:ceph) 2024-07-15T06:37:06.989 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 1 imported monmap: 2024-07-15T06:37:06.989 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr epoch 0 2024-07-15T06:37:06.989 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr fsid 75ba02bc-4274-11ef-bcad-c7b262605968 2024-07-15T06:37:06.989 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr last_changed 2024-07-15T06:37:05.016290+0000 2024-07-15T06:37:06.989 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr created 2024-07-15T06:37:05.016290+0000 2024-07-15T06:37:06.989 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr min_mon_release 15 (octopus) 2024-07-15T06:37:07.018 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr election_strategy: 1 2024-07-15T06:37:07.018 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr 0: [v2:172.21.15.89:3300/0,v1:172.21.15.89:6789/0] mon.a 2024-07-15T06:37:07.018 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr 2024-07-15T06:37:07.018 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 0 /usr/bin/ceph-mon: set fsid to 75ba02bc-4274-11ef-bcad-c7b262605968 2024-07-15T06:37:07.018 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: RocksDB version: 6.15.5 2024-07-15T06:37:07.018 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr 2024-07-15T06:37:07.018 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-07-15T06:37:07.018 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Compile date Jul 14 2024 2024-07-15T06:37:07.018 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: DB SUMMARY 2024-07-15T06:37:07.019 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr 2024-07-15T06:37:07.019 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: DB Session ID: T1BPHGSTMTSJIEJB6BW6 2024-07-15T06:37:07.019 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr 2024-07-15T06:37:07.019 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 0, files: 2024-07-15T06:37:07.019 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr 2024-07-15T06:37:07.019 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 2024-07-15T06:37:07.019 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr 2024-07-15T06:37:07.019 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.error_if_exists: 0 2024-07-15T06:37:07.019 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.create_if_missing: 1 2024-07-15T06:37:07.019 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.paranoid_checks: 1 2024-07-15T06:37:07.019 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-07-15T06:37:07.019 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.env: 0x56143128c440 2024-07-15T06:37:07.019 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.fs: Posix File System 2024-07-15T06:37:07.020 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.info_log: 0x56143289c6a0 2024-07-15T06:37:07.020 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.max_file_opening_threads: 16 2024-07-15T06:37:07.020 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.statistics: (nil) 2024-07-15T06:37:07.020 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.use_fsync: 0 2024-07-15T06:37:07.020 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.max_log_file_size: 0 2024-07-15T06:37:07.020 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.max_manifest_file_size: 1073741824 2024-07-15T06:37:07.020 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.log_file_time_to_roll: 0 2024-07-15T06:37:07.020 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.keep_log_file_num: 1000 2024-07-15T06:37:07.020 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.recycle_log_file_num: 0 2024-07-15T06:37:07.020 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.allow_fallocate: 1 2024-07-15T06:37:07.020 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.allow_mmap_reads: 0 2024-07-15T06:37:07.020 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.allow_mmap_writes: 0 2024-07-15T06:37:07.021 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.use_direct_reads: 0 2024-07-15T06:37:07.021 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-07-15T06:37:07.021 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.create_missing_column_families: 0 2024-07-15T06:37:07.021 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.db_log_dir: 2024-07-15T06:37:07.021 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2024-07-15T06:37:07.021 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.table_cache_numshardbits: 6 2024-07-15T06:37:07.021 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.WAL_ttl_seconds: 0 2024-07-15T06:37:07.021 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.WAL_size_limit_MB: 0 2024-07-15T06:37:07.021 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-07-15T06:37:07.021 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.manifest_preallocation_size: 4194304 2024-07-15T06:37:07.021 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.is_fd_close_on_exec: 1 2024-07-15T06:37:07.021 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.advise_random_on_open: 1 2024-07-15T06:37:07.021 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.db_write_buffer_size: 0 2024-07-15T06:37:07.021 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.write_buffer_manager: 0x5614328e5e60 2024-07-15T06:37:07.022 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.access_hint_on_compaction_start: 1 2024-07-15T06:37:07.022 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-07-15T06:37:07.022 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2024-07-15T06:37:07.022 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.use_adaptive_mutex: 0 2024-07-15T06:37:07.022 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.rate_limiter: (nil) 2024-07-15T06:37:07.022 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-07-15T06:37:07.022 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.wal_recovery_mode: 2 2024-07-15T06:37:07.022 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.enable_thread_tracking: 0 2024-07-15T06:37:07.022 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.enable_pipelined_write: 0 2024-07-15T06:37:07.022 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.unordered_write: 0 2024-07-15T06:37:07.022 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2024-07-15T06:37:07.022 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-07-15T06:37:07.022 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.write_thread_max_yield_usec: 100 2024-07-15T06:37:07.023 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2024-07-15T06:37:07.023 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.row_cache: None 2024-07-15T06:37:07.023 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.wal_filter: None 2024-07-15T06:37:07.023 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.avoid_flush_during_recovery: 0 2024-07-15T06:37:07.023 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.allow_ingest_behind: 0 2024-07-15T06:37:07.023 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.preserve_deletes: 0 2024-07-15T06:37:07.023 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.two_write_queues: 0 2024-07-15T06:37:07.023 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.manual_wal_flush: 0 2024-07-15T06:37:07.023 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.atomic_flush: 0 2024-07-15T06:37:07.023 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-07-15T06:37:07.023 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.persist_stats_to_disk: 0 2024-07-15T06:37:07.023 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.write_dbid_to_manifest: 0 2024-07-15T06:37:07.023 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.log_readahead_size: 0 2024-07-15T06:37:07.024 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.file_checksum_gen_factory: Unknown 2024-07-15T06:37:07.024 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.best_efforts_recovery: 0 2024-07-15T06:37:07.024 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-07-15T06:37:07.024 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-07-15T06:37:07.024 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.allow_data_in_errors: 0 2024-07-15T06:37:07.024 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.db_host_id: __hostname__ 2024-07-15T06:37:07.024 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.max_background_jobs: 2 2024-07-15T06:37:07.024 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.max_background_compactions: -1 2024-07-15T06:37:07.024 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.max_subcompactions: 1 2024-07-15T06:37:07.024 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2024-07-15T06:37:07.024 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-07-15T06:37:07.024 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.delayed_write_rate : 16777216 2024-07-15T06:37:07.024 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.max_total_wal_size: 0 2024-07-15T06:37:07.024 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-07-15T06:37:07.025 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.stats_dump_period_sec: 600 2024-07-15T06:37:07.025 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.stats_persist_period_sec: 600 2024-07-15T06:37:07.025 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.stats_history_buffer_size: 1048576 2024-07-15T06:37:07.025 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.max_open_files: -1 2024-07-15T06:37:07.025 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.bytes_per_sync: 0 2024-07-15T06:37:07.025 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.wal_bytes_per_sync: 0 2024-07-15T06:37:07.025 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.strict_bytes_per_sync: 0 2024-07-15T06:37:07.025 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.compaction_readahead_size: 0 2024-07-15T06:37:07.025 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.max_background_flushes: -1 2024-07-15T06:37:07.025 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Compression algorithms supported: 2024-07-15T06:37:07.025 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: kZSTD supported: 0 2024-07-15T06:37:07.025 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: kXpressCompression supported: 0 2024-07-15T06:37:07.025 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: kBZip2Compression supported: 0 2024-07-15T06:37:07.026 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: kZSTDNotFinalCompression supported: 0 2024-07-15T06:37:07.026 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: kLZ4Compression supported: 1 2024-07-15T06:37:07.026 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: kZlibCompression supported: 1 2024-07-15T06:37:07.026 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: kLZ4HCCompression supported: 1 2024-07-15T06:37:07.026 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: kSnappyCompression supported: 1 2024-07-15T06:37:07.026 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Fast CRC32 supported: Supported on x86 2024-07-15T06:37:07.026 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: [db/db_impl/db_impl_open.cc:281] Creating manifest 1 2024-07-15T06:37:07.026 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr 2024-07-15T06:37:07.026 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000001 2024-07-15T06:37:07.026 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr 2024-07-15T06:37:07.026 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2024-07-15T06:37:07.026 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr 2024-07-15T06:37:07.026 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-07-15T06:37:07.027 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.merge_operator: 2024-07-15T06:37:07.027 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.compaction_filter: None 2024-07-15T06:37:07.027 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.compaction_filter_factory: None 2024-07-15T06:37:07.027 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.sst_partitioner_factory: None 2024-07-15T06:37:07.027 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.memtable_factory: SkipListFactory 2024-07-15T06:37:07.027 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.table_factory: BlockBasedTable 2024-07-15T06:37:07.027 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x56143283dea8) 2024-07-15T06:37:07.027 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr cache_index_and_filter_blocks: 1 2024-07-15T06:37:07.027 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr cache_index_and_filter_blocks_with_high_priority: 0 2024-07-15T06:37:07.027 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr pin_l0_filter_and_index_blocks_in_cache: 0 2024-07-15T06:37:07.027 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr pin_top_level_index_and_filter: 1 2024-07-15T06:37:07.027 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr index_type: 0 2024-07-15T06:37:07.027 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr data_block_index_type: 0 2024-07-15T06:37:07.027 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr index_shortening: 1 2024-07-15T06:37:07.028 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr data_block_hash_table_util_ratio: 0.750000 2024-07-15T06:37:07.028 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr hash_index_allow_collision: 1 2024-07-15T06:37:07.028 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr checksum: 1 2024-07-15T06:37:07.028 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr no_block_cache: 0 2024-07-15T06:37:07.028 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr block_cache: 0x5614328751f0 2024-07-15T06:37:07.028 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr block_cache_name: BinnedLRUCache 2024-07-15T06:37:07.028 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr block_cache_options: 2024-07-15T06:37:07.028 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr capacity : 536870912 2024-07-15T06:37:07.028 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr num_shard_bits : 4 2024-07-15T06:37:07.028 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr strict_capacity_limit : 0 2024-07-15T06:37:07.028 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr high_pri_pool_ratio: 0.000 2024-07-15T06:37:07.028 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr block_cache_compressed: (nil) 2024-07-15T06:37:07.028 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr persistent_cache: (nil) 2024-07-15T06:37:07.028 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr block_size: 4096 2024-07-15T06:37:07.029 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr block_size_deviation: 10 2024-07-15T06:37:07.029 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr block_restart_interval: 16 2024-07-15T06:37:07.029 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr index_block_restart_interval: 1 2024-07-15T06:37:07.029 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr metadata_block_size: 4096 2024-07-15T06:37:07.029 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr partition_filters: 0 2024-07-15T06:37:07.029 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr use_delta_encoding: 1 2024-07-15T06:37:07.029 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr filter_policy: rocksdb.BuiltinBloomFilter 2024-07-15T06:37:07.029 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr whole_key_filtering: 1 2024-07-15T06:37:07.029 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr verify_compression: 0 2024-07-15T06:37:07.029 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr read_amp_bytes_per_bit: 0 2024-07-15T06:37:07.029 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr format_version: 4 2024-07-15T06:37:07.029 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr enable_index_compression: 1 2024-07-15T06:37:07.029 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr block_align: 0 2024-07-15T06:37:07.029 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr 2024-07-15T06:37:07.030 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.write_buffer_size: 33554432 2024-07-15T06:37:07.030 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.max_write_buffer_number: 2 2024-07-15T06:37:07.030 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.compression: NoCompression 2024-07-15T06:37:07.030 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.bottommost_compression: Disabled 2024-07-15T06:37:07.030 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.prefix_extractor: nullptr 2024-07-15T06:37:07.030 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-07-15T06:37:07.030 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.num_levels: 7 2024-07-15T06:37:07.030 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-07-15T06:37:07.030 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-07-15T06:37:07.030 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-07-15T06:37:07.030 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-07-15T06:37:07.030 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2024-07-15T06:37:07.030 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-07-15T06:37:07.030 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-07-15T06:37:07.031 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-07-15T06:37:07.031 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-07-15T06:37:07.031 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2024-07-15T06:37:07.031 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.compression_opts.window_bits: -14 2024-07-15T06:37:07.031 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.compression_opts.level: 32767 2024-07-15T06:37:07.031 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.compression_opts.strategy: 0 2024-07-15T06:37:07.031 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-07-15T06:37:07.031 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-07-15T06:37:07.031 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.compression_opts.parallel_threads: 1 2024-07-15T06:37:07.031 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.compression_opts.enabled: false 2024-07-15T06:37:07.031 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-07-15T06:37:07.031 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-07-15T06:37:07.031 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.level0_stop_writes_trigger: 36 2024-07-15T06:37:07.032 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.target_file_size_base: 67108864 2024-07-15T06:37:07.032 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.target_file_size_multiplier: 1 2024-07-15T06:37:07.032 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2024-07-15T06:37:07.032 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-07-15T06:37:07.032 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-07-15T06:37:07.032 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-07-15T06:37:07.032 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-07-15T06:37:07.032 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-07-15T06:37:07.032 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-07-15T06:37:07.032 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-07-15T06:37:07.032 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-07-15T06:37:07.032 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-07-15T06:37:07.032 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-07-15T06:37:07.033 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.max_compaction_bytes: 1677721600 2024-07-15T06:37:07.033 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.arena_block_size: 4194304 2024-07-15T06:37:07.033 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-07-15T06:37:07.033 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-07-15T06:37:07.033 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-07-15T06:37:07.033 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.disable_auto_compactions: 0 2024-07-15T06:37:07.033 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-07-15T06:37:07.033 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-07-15T06:37:07.033 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-07-15T06:37:07.033 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-07-15T06:37:07.033 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-07-15T06:37:07.033 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-07-15T06:37:07.033 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-07-15T06:37:07.034 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-07-15T06:37:07.034 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-07-15T06:37:07.034 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-07-15T06:37:07.034 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.table_properties_collectors: 2024-07-15T06:37:07.034 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.inplace_update_support: 0 2024-07-15T06:37:07.034 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.inplace_update_num_locks: 10000 2024-07-15T06:37:07.034 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-07-15T06:37:07.034 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.memtable_whole_key_filtering: 0 2024-07-15T06:37:07.034 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.memtable_huge_page_size: 0 2024-07-15T06:37:07.034 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.bloom_locality: 0 2024-07-15T06:37:07.034 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.max_successive_merges: 0 2024-07-15T06:37:07.034 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.optimize_filters_for_hits: 0 2024-07-15T06:37:07.034 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.paranoid_file_checks: 0 2024-07-15T06:37:07.034 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.force_consistency_checks: 1 2024-07-15T06:37:07.035 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.report_bg_io_stats: 0 2024-07-15T06:37:07.035 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.ttl: 2592000 2024-07-15T06:37:07.035 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.periodic_compaction_seconds: 0 2024-07-15T06:37:07.035 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.enable_blob_files: false 2024-07-15T06:37:07.035 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.min_blob_size: 0 2024-07-15T06:37:07.035 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.blob_file_size: 268435456 2024-07-15T06:37:07.035 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.blob_compression_type: NoCompression 2024-07-15T06:37:07.035 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.enable_blob_garbage_collection: false 2024-07-15T06:37:07.035 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.468+0000 7f662482e900 4 rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-07-15T06:37:07.035 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.472+0000 7f662482e900 4 rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000001 succeeded,manifest_file_number is 1, next_file_number is 3, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-07-15T06:37:07.035 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr 2024-07-15T06:37:07.035 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.472+0000 7f662482e900 4 rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 0 2024-07-15T06:37:07.035 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr 2024-07-15T06:37:07.035 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.472+0000 7f662482e900 4 rocksdb: [db/version_set.cc:4082] Creating manifest 3 2024-07-15T06:37:07.036 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr 2024-07-15T06:37:07.036 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.472+0000 7f662482e900 4 rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x5614328a6700 2024-07-15T06:37:07.036 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.472+0000 7f662482e900 4 rocksdb: DB pointer 0x561432878000 2024-07-15T06:37:07.036 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.472+0000 7f661bfbc640 4 rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2024-07-15T06:37:07.036 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.472+0000 7f661bfbc640 4 rocksdb: [db/db_impl/db_impl.cc:903] 2024-07-15T06:37:07.036 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr ** DB Stats ** 2024-07-15T06:37:07.036 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr Uptime(secs): 0.0 total, 0.0 interval 2024-07-15T06:37:07.036 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-07-15T06:37:07.036 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-07-15T06:37:07.036 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-07-15T06:37:07.036 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-07-15T06:37:07.036 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-07-15T06:37:07.036 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-07-15T06:37:07.036 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr 2024-07-15T06:37:07.036 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr ** Compaction Stats [default] ** 2024-07-15T06:37:07.037 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-07-15T06:37:07.037 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-07-15T06:37:07.037 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr Sum 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-07-15T06:37:07.037 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-07-15T06:37:07.037 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr 2024-07-15T06:37:07.037 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr ** Compaction Stats [default] ** 2024-07-15T06:37:07.037 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-07-15T06:37:07.037 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-07-15T06:37:07.037 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr Uptime(secs): 0.0 total, 0.0 interval 2024-07-15T06:37:07.037 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr Flush(GB): cumulative 0.000, interval 0.000 2024-07-15T06:37:07.037 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr AddFile(GB): cumulative 0.000, interval 0.000 2024-07-15T06:37:07.037 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr AddFile(Total Files): cumulative 0, interval 0 2024-07-15T06:37:07.037 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr AddFile(L0 Files): cumulative 0, interval 0 2024-07-15T06:37:07.037 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr AddFile(Keys): cumulative 0, interval 0 2024-07-15T06:37:07.038 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-07-15T06:37:07.038 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-07-15T06:37:07.038 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-07-15T06:37:07.038 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr 2024-07-15T06:37:07.038 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr ** File Read Latency Histogram By Level [default] ** 2024-07-15T06:37:07.038 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr 2024-07-15T06:37:07.038 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr ** Compaction Stats [default] ** 2024-07-15T06:37:07.038 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-07-15T06:37:07.038 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-07-15T06:37:07.038 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr Sum 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-07-15T06:37:07.038 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-07-15T06:37:07.038 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr 2024-07-15T06:37:07.039 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr ** Compaction Stats [default] ** 2024-07-15T06:37:07.039 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-07-15T06:37:07.039 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-07-15T06:37:07.039 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr Uptime(secs): 0.0 total, 0.0 interval 2024-07-15T06:37:07.039 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr Flush(GB): cumulative 0.000, interval 0.000 2024-07-15T06:37:07.039 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr AddFile(GB): cumulative 0.000, interval 0.000 2024-07-15T06:37:07.039 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr AddFile(Total Files): cumulative 0, interval 0 2024-07-15T06:37:07.039 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr AddFile(L0 Files): cumulative 0, interval 0 2024-07-15T06:37:07.039 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr AddFile(Keys): cumulative 0, interval 0 2024-07-15T06:37:07.039 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-07-15T06:37:07.039 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-07-15T06:37:07.039 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-07-15T06:37:07.039 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr 2024-07-15T06:37:07.039 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr ** File Read Latency Histogram By Level [default] ** 2024-07-15T06:37:07.039 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr 2024-07-15T06:37:07.040 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.472+0000 7f662482e900 4 rocksdb: [db/db_impl/db_impl.cc:446] Shutdown: canceling all background work 2024-07-15T06:37:07.040 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.472+0000 7f662482e900 4 rocksdb: [db/db_impl/db_impl.cc:625] Shutdown complete 2024-07-15T06:37:07.040 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-15T06:37:06.472+0000 7f662482e900 0 /usr/bin/ceph-mon: created monfs at /var/lib/ceph/mon/ceph-a for mon.a 2024-07-15T06:37:07.040 INFO:teuthology.orchestra.run.smithi089.stdout:create mon.a on 2024-07-15T06:37:07.279 INFO:teuthology.orchestra.run.smithi089.stdout:systemctl: stderr Removed /etc/systemd/system/multi-user.target.wants/ceph.target. 2024-07-15T06:37:07.488 INFO:teuthology.orchestra.run.smithi089.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target -> /etc/systemd/system/ceph.target. 2024-07-15T06:37:07.664 INFO:teuthology.orchestra.run.smithi089.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph-75ba02bc-4274-11ef-bcad-c7b262605968.target -> /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968.target. 2024-07-15T06:37:07.664 INFO:teuthology.orchestra.run.smithi089.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph.target.wants/ceph-75ba02bc-4274-11ef-bcad-c7b262605968.target -> /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968.target. 2024-07-15T06:37:07.882 INFO:teuthology.orchestra.run.smithi089.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-75ba02bc-4274-11ef-bcad-c7b262605968@mon.a 2024-07-15T06:37:07.883 INFO:teuthology.orchestra.run.smithi089.stdout:systemctl: stderr Failed to reset failed state of unit ceph-75ba02bc-4274-11ef-bcad-c7b262605968@mon.a.service: Unit ceph-75ba02bc-4274-11ef-bcad-c7b262605968@mon.a.service not loaded. 2024-07-15T06:37:08.098 INFO:teuthology.orchestra.run.smithi089.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968.target.wants/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@mon.a.service -> /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service. 2024-07-15T06:37:08.107 INFO:teuthology.orchestra.run.smithi089.stdout:firewalld does not appear to be present 2024-07-15T06:37:08.107 INFO:teuthology.orchestra.run.smithi089.stdout:Not possible to enable service . firewalld.service is not available 2024-07-15T06:37:08.107 INFO:teuthology.orchestra.run.smithi089.stdout:Waiting for mon to start... 2024-07-15T06:37:08.107 INFO:teuthology.orchestra.run.smithi089.stdout:Waiting for mon... 2024-07-15T06:37:08.217 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:08 smithi089 systemd[1]: Started Ceph mon.a for 75ba02bc-4274-11ef-bcad-c7b262605968. 2024-07-15T06:37:09.862 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:09 smithi089 bash[21307]: cluster 2024-07-15T06:37:09.635043+0000 mon.a (mon.0) 0 : cluster [INF] mkfs 75ba02bc-4274-11ef-bcad-c7b262605968 2024-07-15T06:37:09.862 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:09 smithi089 bash[21307]: cluster 2024-07-15T06:37:09.628946+0000 mon.a (mon.0) 1 : cluster [INF] mon.a is new leader, mons a in quorum (ranks 0) 2024-07-15T06:37:10.392 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout cluster: 2024-07-15T06:37:10.392 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout id: 75ba02bc-4274-11ef-bcad-c7b262605968 2024-07-15T06:37:10.392 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout health: HEALTH_OK 2024-07-15T06:37:10.392 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout 2024-07-15T06:37:10.392 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout services: 2024-07-15T06:37:10.392 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout mon: 1 daemons, quorum a (age 0.220291s) 2024-07-15T06:37:10.392 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout mgr: no daemons active 2024-07-15T06:37:10.392 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout osd: 0 osds: 0 up, 0 in 2024-07-15T06:37:10.392 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout 2024-07-15T06:37:10.392 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout data: 2024-07-15T06:37:10.392 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout pools: 0 pools, 0 pgs 2024-07-15T06:37:10.393 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout objects: 0 objects, 0 B 2024-07-15T06:37:10.393 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout usage: 0 B used, 0 B / 0 B avail 2024-07-15T06:37:10.393 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout pgs: 2024-07-15T06:37:10.393 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout 2024-07-15T06:37:10.393 INFO:teuthology.orchestra.run.smithi089.stdout:mon is available 2024-07-15T06:37:10.393 INFO:teuthology.orchestra.run.smithi089.stdout:Assimilating anything we can from ceph.conf... 2024-07-15T06:37:10.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:10 smithi089 bash[21307]: cluster 2024-07-15T06:37:09.633883+0000 mon.a (mon.0) 2 : cluster [INF] mon.a is new leader, mons a in quorum (ranks 0) 2024-07-15T06:37:10.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:10 smithi089 bash[21307]: cluster 2024-07-15T06:37:09.634097+0000 mon.a (mon.0) 3 : cluster [DBG] monmap e1: 1 mons at {a=[v2:172.21.15.89:3300/0,v1:172.21.15.89:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-07-15T06:37:10.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:10 smithi089 bash[21307]: cluster 2024-07-15T06:37:09.636561+0000 mon.a (mon.0) 4 : cluster [DBG] fsmap 2024-07-15T06:37:10.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:10 smithi089 bash[21307]: cluster 2024-07-15T06:37:09.638863+0000 mon.a (mon.0) 5 : cluster [DBG] osdmap e1: 0 total, 0 up, 0 in 2024-07-15T06:37:10.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:10 smithi089 bash[21307]: cluster 2024-07-15T06:37:09.639395+0000 mon.a (mon.0) 6 : cluster [DBG] mgrmap e1: no daemons active 2024-07-15T06:37:10.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:10 smithi089 bash[21307]: audit 2024-07-15T06:37:09.854144+0000 mon.a (mon.0) 7 : audit [DBG] from='client.? 172.21.15.89:0/4150777049' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-07-15T06:37:11.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:11 smithi089 bash[21307]: audit 2024-07-15T06:37:11.511631+0000 mon.a (mon.0) 8 : audit [INF] from='client.? 172.21.15.89:0/3325209453' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-07-15T06:37:11.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:11 smithi089 bash[21307]: audit 2024-07-15T06:37:11.515171+0000 mon.a (mon.0) 9 : audit [INF] from='client.? 172.21.15.89:0/3325209453' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-07-15T06:37:12.107 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout 2024-07-15T06:37:12.107 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout [global] 2024-07-15T06:37:12.107 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout fsid = 75ba02bc-4274-11ef-bcad-c7b262605968 2024-07-15T06:37:12.108 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout mon_host = [v2:172.21.15.89:3300,v1:172.21.15.89:6789] 2024-07-15T06:37:12.108 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2024-07-15T06:37:12.108 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2024-07-15T06:37:12.108 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2024-07-15T06:37:12.108 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2024-07-15T06:37:12.108 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout 2024-07-15T06:37:12.108 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout [mgr] 2024-07-15T06:37:12.108 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2024-07-15T06:37:12.108 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout 2024-07-15T06:37:12.108 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout [osd] 2024-07-15T06:37:12.108 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2024-07-15T06:37:12.109 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2024-07-15T06:37:12.109 INFO:teuthology.orchestra.run.smithi089.stdout:Generating new minimal ceph.conf... 2024-07-15T06:37:13.622 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:13 smithi089 bash[21307]: audit 2024-07-15T06:37:13.268612+0000 mon.a (mon.0) 10 : audit [DBG] from='client.? 172.21.15.89:0/3485484505' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:37:13.737 INFO:teuthology.orchestra.run.smithi089.stdout:Restarting the monitor... 2024-07-15T06:37:14.072 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:13 smithi089 systemd[1]: Stopping Ceph mon.a for 75ba02bc-4274-11ef-bcad-c7b262605968... 2024-07-15T06:37:14.072 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:13 smithi089 bash[21307]: debug 2024-07-15T06:37:13.844+0000 7ff9a8060640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2024-07-15T06:37:14.072 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:13 smithi089 bash[21307]: debug 2024-07-15T06:37:13.844+0000 7ff9a8060640 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2024-07-15T06:37:14.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:14 smithi089 bash[21691]: ceph-75ba02bc-4274-11ef-bcad-c7b262605968-mon-a 2024-07-15T06:37:14.433 INFO:teuthology.orchestra.run.smithi089.stdout:Setting public_network to 172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32 in mon config section 2024-07-15T06:37:14.774 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:14 smithi089 bash[21753]: Error response from daemon: No such container: ceph-75ba02bc-4274-11ef-bcad-c7b262605968-mon-a 2024-07-15T06:37:14.774 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:14 smithi089 systemd[1]: ceph-75ba02bc-4274-11ef-bcad-c7b262605968@mon.a.service: Deactivated successfully. 2024-07-15T06:37:14.774 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:14 smithi089 systemd[1]: Stopped Ceph mon.a for 75ba02bc-4274-11ef-bcad-c7b262605968. 2024-07-15T06:37:14.774 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:14 smithi089 systemd[1]: Started Ceph mon.a for 75ba02bc-4274-11ef-bcad-c7b262605968. 2024-07-15T06:37:16.087 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.860+0000 7fb0c3043900 0 set uid:gid to 167:167 (ceph:ceph) 2024-07-15T06:37:16.087 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.860+0000 7fb0c3043900 0 ceph version 17.2.7-1102-g9a202024 (9a2020246f85463d4fdb115c94aa40b463caa1d2) quincy (stable), process ceph-mon, pid 7 2024-07-15T06:37:16.087 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.860+0000 7fb0c3043900 0 pidfile_write: ignore empty --pid-file 2024-07-15T06:37:16.088 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 0 load: jerasure load: lrc 2024-07-15T06:37:16.088 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: RocksDB version: 6.15.5 2024-07-15T06:37:16.088 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-07-15T06:37:16.088 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Compile date Jul 14 2024 2024-07-15T06:37:16.088 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: DB SUMMARY 2024-07-15T06:37:16.088 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: DB Session ID: A1NZWW83ZPSHN9BMK1U2 2024-07-15T06:37:16.088 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: CURRENT file: CURRENT 2024-07-15T06:37:16.088 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: IDENTITY file: IDENTITY 2024-07-15T06:37:16.088 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2024-07-15T06:37:16.088 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000008.sst 2024-07-15T06:37:16.088 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000010.log size: 83713 ; 2024-07-15T06:37:16.088 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.error_if_exists: 0 2024-07-15T06:37:16.089 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.create_if_missing: 0 2024-07-15T06:37:16.089 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.paranoid_checks: 1 2024-07-15T06:37:16.089 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-07-15T06:37:16.089 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.env: 0x559b47351440 2024-07-15T06:37:16.089 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.fs: Posix File System 2024-07-15T06:37:16.089 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.info_log: 0x559b49894f40 2024-07-15T06:37:16.089 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.max_file_opening_threads: 16 2024-07-15T06:37:16.089 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.statistics: (nil) 2024-07-15T06:37:16.089 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.use_fsync: 0 2024-07-15T06:37:16.089 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.max_log_file_size: 0 2024-07-15T06:37:16.089 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.max_manifest_file_size: 1073741824 2024-07-15T06:37:16.089 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.log_file_time_to_roll: 0 2024-07-15T06:37:16.091 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.keep_log_file_num: 1000 2024-07-15T06:37:16.091 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.recycle_log_file_num: 0 2024-07-15T06:37:16.091 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.allow_fallocate: 1 2024-07-15T06:37:16.091 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.allow_mmap_reads: 0 2024-07-15T06:37:16.091 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.allow_mmap_writes: 0 2024-07-15T06:37:16.091 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.use_direct_reads: 0 2024-07-15T06:37:16.092 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-07-15T06:37:16.092 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.create_missing_column_families: 0 2024-07-15T06:37:16.092 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.db_log_dir: 2024-07-15T06:37:16.092 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2024-07-15T06:37:16.092 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.table_cache_numshardbits: 6 2024-07-15T06:37:16.092 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.WAL_ttl_seconds: 0 2024-07-15T06:37:16.092 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.WAL_size_limit_MB: 0 2024-07-15T06:37:16.092 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-07-15T06:37:16.092 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.manifest_preallocation_size: 4194304 2024-07-15T06:37:16.092 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.is_fd_close_on_exec: 1 2024-07-15T06:37:16.092 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.advise_random_on_open: 1 2024-07-15T06:37:16.092 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.db_write_buffer_size: 0 2024-07-15T06:37:16.092 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.write_buffer_manager: 0x559b499527b0 2024-07-15T06:37:16.093 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.access_hint_on_compaction_start: 1 2024-07-15T06:37:16.093 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-07-15T06:37:16.093 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2024-07-15T06:37:16.093 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.use_adaptive_mutex: 0 2024-07-15T06:37:16.093 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.rate_limiter: (nil) 2024-07-15T06:37:16.093 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-07-15T06:37:16.093 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.wal_recovery_mode: 2 2024-07-15T06:37:16.093 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.enable_thread_tracking: 0 2024-07-15T06:37:16.093 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.enable_pipelined_write: 0 2024-07-15T06:37:16.093 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.unordered_write: 0 2024-07-15T06:37:16.093 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2024-07-15T06:37:16.093 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-07-15T06:37:16.093 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.write_thread_max_yield_usec: 100 2024-07-15T06:37:16.094 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2024-07-15T06:37:16.094 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.row_cache: None 2024-07-15T06:37:16.094 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.wal_filter: None 2024-07-15T06:37:16.094 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.avoid_flush_during_recovery: 0 2024-07-15T06:37:16.094 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.allow_ingest_behind: 0 2024-07-15T06:37:16.094 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.preserve_deletes: 0 2024-07-15T06:37:16.094 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.two_write_queues: 0 2024-07-15T06:37:16.094 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.manual_wal_flush: 0 2024-07-15T06:37:16.094 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.atomic_flush: 0 2024-07-15T06:37:16.094 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-07-15T06:37:16.094 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.persist_stats_to_disk: 0 2024-07-15T06:37:16.094 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.write_dbid_to_manifest: 0 2024-07-15T06:37:16.094 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.log_readahead_size: 0 2024-07-15T06:37:16.095 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.file_checksum_gen_factory: Unknown 2024-07-15T06:37:16.095 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.best_efforts_recovery: 0 2024-07-15T06:37:16.095 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-07-15T06:37:16.095 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-07-15T06:37:16.095 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.allow_data_in_errors: 0 2024-07-15T06:37:16.095 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.db_host_id: __hostname__ 2024-07-15T06:37:16.095 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.max_background_jobs: 2 2024-07-15T06:37:16.095 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.max_background_compactions: -1 2024-07-15T06:37:16.095 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.max_subcompactions: 1 2024-07-15T06:37:16.095 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2024-07-15T06:37:16.095 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-07-15T06:37:16.095 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.delayed_write_rate : 16777216 2024-07-15T06:37:16.095 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.max_total_wal_size: 0 2024-07-15T06:37:16.096 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-07-15T06:37:16.096 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.stats_dump_period_sec: 600 2024-07-15T06:37:16.096 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.stats_persist_period_sec: 600 2024-07-15T06:37:16.096 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.stats_history_buffer_size: 1048576 2024-07-15T06:37:16.096 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.max_open_files: -1 2024-07-15T06:37:16.096 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.bytes_per_sync: 0 2024-07-15T06:37:16.096 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.wal_bytes_per_sync: 0 2024-07-15T06:37:16.096 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.strict_bytes_per_sync: 0 2024-07-15T06:37:16.096 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.compaction_readahead_size: 0 2024-07-15T06:37:16.096 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.max_background_flushes: -1 2024-07-15T06:37:16.096 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Compression algorithms supported: 2024-07-15T06:37:16.096 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: kZSTD supported: 0 2024-07-15T06:37:16.096 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: kXpressCompression supported: 0 2024-07-15T06:37:16.096 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: kBZip2Compression supported: 0 2024-07-15T06:37:16.097 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: kZSTDNotFinalCompression supported: 0 2024-07-15T06:37:16.097 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: kLZ4Compression supported: 1 2024-07-15T06:37:16.097 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: kZlibCompression supported: 1 2024-07-15T06:37:16.097 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: kLZ4HCCompression supported: 1 2024-07-15T06:37:16.097 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: kSnappyCompression supported: 1 2024-07-15T06:37:16.097 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Fast CRC32 supported: Supported on x86 2024-07-15T06:37:16.097 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 2024-07-15T06:37:16.097 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2024-07-15T06:37:16.097 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-07-15T06:37:16.097 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.merge_operator: 2024-07-15T06:37:16.097 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.compaction_filter: None 2024-07-15T06:37:16.097 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.compaction_filter_factory: None 2024-07-15T06:37:16.097 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.sst_partitioner_factory: None 2024-07-15T06:37:16.098 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.memtable_factory: SkipListFactory 2024-07-15T06:37:16.098 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.table_factory: BlockBasedTable 2024-07-15T06:37:16.098 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x559b4983da68) 2024-07-15T06:37:16.098 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: cache_index_and_filter_blocks: 1 2024-07-15T06:37:16.098 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: cache_index_and_filter_blocks_with_high_priority: 0 2024-07-15T06:37:16.098 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: pin_l0_filter_and_index_blocks_in_cache: 0 2024-07-15T06:37:16.098 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: pin_top_level_index_and_filter: 1 2024-07-15T06:37:16.098 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: index_type: 0 2024-07-15T06:37:16.098 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: data_block_index_type: 0 2024-07-15T06:37:16.098 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: index_shortening: 1 2024-07-15T06:37:16.098 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: data_block_hash_table_util_ratio: 0.750000 2024-07-15T06:37:16.098 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: hash_index_allow_collision: 1 2024-07-15T06:37:16.098 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: checksum: 1 2024-07-15T06:37:16.099 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: no_block_cache: 0 2024-07-15T06:37:16.099 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: block_cache: 0x559b49871350 2024-07-15T06:37:16.099 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: block_cache_name: BinnedLRUCache 2024-07-15T06:37:16.099 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: block_cache_options: 2024-07-15T06:37:16.099 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: capacity : 536870912 2024-07-15T06:37:16.099 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: num_shard_bits : 4 2024-07-15T06:37:16.099 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: strict_capacity_limit : 0 2024-07-15T06:37:16.099 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: high_pri_pool_ratio: 0.000 2024-07-15T06:37:16.099 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: block_cache_compressed: (nil) 2024-07-15T06:37:16.099 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: persistent_cache: (nil) 2024-07-15T06:37:16.099 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: block_size: 4096 2024-07-15T06:37:16.099 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: block_size_deviation: 10 2024-07-15T06:37:16.099 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: block_restart_interval: 16 2024-07-15T06:37:16.100 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: index_block_restart_interval: 1 2024-07-15T06:37:16.100 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: metadata_block_size: 4096 2024-07-15T06:37:16.100 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: partition_filters: 0 2024-07-15T06:37:16.100 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: use_delta_encoding: 1 2024-07-15T06:37:16.100 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: filter_policy: rocksdb.BuiltinBloomFilter 2024-07-15T06:37:16.100 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: whole_key_filtering: 1 2024-07-15T06:37:16.100 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: verify_compression: 0 2024-07-15T06:37:16.100 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: read_amp_bytes_per_bit: 0 2024-07-15T06:37:16.100 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: format_version: 4 2024-07-15T06:37:16.100 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: enable_index_compression: 1 2024-07-15T06:37:16.100 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: block_align: 0 2024-07-15T06:37:16.100 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.write_buffer_size: 33554432 2024-07-15T06:37:16.100 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.max_write_buffer_number: 2 2024-07-15T06:37:16.101 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.compression: NoCompression 2024-07-15T06:37:16.101 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.bottommost_compression: Disabled 2024-07-15T06:37:16.101 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.prefix_extractor: nullptr 2024-07-15T06:37:16.101 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-07-15T06:37:16.101 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.num_levels: 7 2024-07-15T06:37:16.101 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-07-15T06:37:16.101 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-07-15T06:37:16.101 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-07-15T06:37:16.101 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-07-15T06:37:16.101 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2024-07-15T06:37:16.101 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-07-15T06:37:16.101 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-07-15T06:37:16.101 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-07-15T06:37:16.101 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-07-15T06:37:16.102 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2024-07-15T06:37:16.102 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.compression_opts.window_bits: -14 2024-07-15T06:37:16.102 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.compression_opts.level: 32767 2024-07-15T06:37:16.102 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.compression_opts.strategy: 0 2024-07-15T06:37:16.102 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-07-15T06:37:16.102 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-07-15T06:37:16.102 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.compression_opts.parallel_threads: 1 2024-07-15T06:37:16.102 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.compression_opts.enabled: false 2024-07-15T06:37:16.102 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-07-15T06:37:16.102 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-07-15T06:37:16.102 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.level0_stop_writes_trigger: 36 2024-07-15T06:37:16.102 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.target_file_size_base: 67108864 2024-07-15T06:37:16.102 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.target_file_size_multiplier: 1 2024-07-15T06:37:16.103 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2024-07-15T06:37:16.103 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-07-15T06:37:16.103 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-07-15T06:37:16.103 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-07-15T06:37:16.103 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-07-15T06:37:16.103 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-07-15T06:37:16.103 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-07-15T06:37:16.103 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-07-15T06:37:16.103 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-07-15T06:37:16.103 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-07-15T06:37:16.103 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-07-15T06:37:16.103 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.max_compaction_bytes: 1677721600 2024-07-15T06:37:16.103 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.arena_block_size: 4194304 2024-07-15T06:37:16.103 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-07-15T06:37:16.104 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-07-15T06:37:16.104 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-07-15T06:37:16.104 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.disable_auto_compactions: 0 2024-07-15T06:37:16.104 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-07-15T06:37:16.104 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-07-15T06:37:16.104 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-07-15T06:37:16.104 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-07-15T06:37:16.104 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-07-15T06:37:16.104 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-07-15T06:37:16.104 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-07-15T06:37:16.105 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-07-15T06:37:16.105 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-07-15T06:37:16.105 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-07-15T06:37:16.105 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.table_properties_collectors: 2024-07-15T06:37:16.105 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.inplace_update_support: 0 2024-07-15T06:37:16.105 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.inplace_update_num_locks: 10000 2024-07-15T06:37:16.105 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-07-15T06:37:16.105 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.memtable_whole_key_filtering: 0 2024-07-15T06:37:16.105 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.memtable_huge_page_size: 0 2024-07-15T06:37:16.105 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.bloom_locality: 0 2024-07-15T06:37:16.105 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.max_successive_merges: 0 2024-07-15T06:37:16.105 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.optimize_filters_for_hits: 0 2024-07-15T06:37:16.105 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.paranoid_file_checks: 0 2024-07-15T06:37:16.106 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.force_consistency_checks: 1 2024-07-15T06:37:16.106 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.report_bg_io_stats: 0 2024-07-15T06:37:16.106 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.ttl: 2592000 2024-07-15T06:37:16.106 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.periodic_compaction_seconds: 0 2024-07-15T06:37:16.106 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.enable_blob_files: false 2024-07-15T06:37:16.106 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.min_blob_size: 0 2024-07-15T06:37:16.106 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.blob_file_size: 268435456 2024-07-15T06:37:16.106 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.blob_compression_type: NoCompression 2024-07-15T06:37:16.106 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.enable_blob_garbage_collection: false 2024-07-15T06:37:16.106 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.864+0000 7fb0c3043900 4 rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-07-15T06:37:16.106 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.868+0000 7fb0c3043900 4 rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-07-15T06:37:16.106 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.868+0000 7fb0c3043900 4 rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 5 2024-07-15T06:37:16.107 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.868+0000 7fb0c3043900 4 rocksdb: [db/version_set.cc:4082] Creating manifest 13 2024-07-15T06:37:16.107 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.868+0000 7fb0c3043900 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1721025435872335, "job": 1, "event": "recovery_started", "wal_files": [10]} 2024-07-15T06:37:16.107 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.868+0000 7fb0c3043900 4 rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #10 mode 2 2024-07-15T06:37:16.107 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.868+0000 7fb0c3043900 3 rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-07-15T06:37:16.107 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.868+0000 7fb0c3043900 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1721025435873614, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 80260, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 78528, "index_size": 220, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 10136, "raw_average_key_size": 47, "raw_value_size": 72736, "raw_average_value_size": 341, "num_data_blocks": 10, "num_entries": 213, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1721025435, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "8bedf614-7361-410b-bc7e-ccbb3003ec0a", "db_session_id": "A1NZWW83ZPSHN9BMK1U2"}} 2024-07-15T06:37:16.107 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.868+0000 7fb0c3043900 4 rocksdb: [db/version_set.cc:4082] Creating manifest 15 2024-07-15T06:37:16.107 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.868+0000 7fb0c3043900 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1721025435874161, "job": 1, "event": "recovery_finished"} 2024-07-15T06:37:16.107 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.868+0000 7fb0c3043900 4 rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-07-15T06:37:16.107 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.872+0000 7fb0c3043900 4 rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x559b498a6700 2024-07-15T06:37:16.107 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.872+0000 7fb0c3043900 4 rocksdb: DB pointer 0x559b49874000 2024-07-15T06:37:16.107 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.872+0000 7fb0b8e0f640 4 rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2024-07-15T06:37:16.107 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.872+0000 7fb0b8e0f640 4 rocksdb: [db/db_impl/db_impl.cc:903] 2024-07-15T06:37:16.107 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: ** DB Stats ** 2024-07-15T06:37:16.107 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: Uptime(secs): 0.0 total, 0.0 interval 2024-07-15T06:37:16.108 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-07-15T06:37:16.108 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-07-15T06:37:16.108 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-07-15T06:37:16.108 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-07-15T06:37:16.108 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-07-15T06:37:16.108 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-07-15T06:37:16.108 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: ** Compaction Stats [default] ** 2024-07-15T06:37:16.108 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-07-15T06:37:16.108 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-07-15T06:37:16.108 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: L0 2/0 80.14 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 92.7 0.00 0.00 1 0.001 0 0 2024-07-15T06:37:16.108 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: Sum 2/0 80.14 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 92.7 0.00 0.00 1 0.001 0 0 2024-07-15T06:37:16.108 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 92.7 0.00 0.00 1 0.001 0 0 2024-07-15T06:37:16.109 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: ** Compaction Stats [default] ** 2024-07-15T06:37:16.109 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-07-15T06:37:16.109 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-07-15T06:37:16.109 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 92.7 0.00 0.00 1 0.001 0 0 2024-07-15T06:37:16.109 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: Uptime(secs): 0.0 total, 0.0 interval 2024-07-15T06:37:16.109 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: Flush(GB): cumulative 0.000, interval 0.000 2024-07-15T06:37:16.109 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: AddFile(GB): cumulative 0.000, interval 0.000 2024-07-15T06:37:16.109 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: AddFile(Total Files): cumulative 0, interval 0 2024-07-15T06:37:16.109 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: AddFile(L0 Files): cumulative 0, interval 0 2024-07-15T06:37:16.109 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: AddFile(Keys): cumulative 0, interval 0 2024-07-15T06:37:16.109 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: Cumulative compaction: 0.00 GB write, 14.02 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-07-15T06:37:16.109 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: Interval compaction: 0.00 GB write, 14.02 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-07-15T06:37:16.109 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-07-15T06:37:16.110 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: ** File Read Latency Histogram By Level [default] ** 2024-07-15T06:37:16.110 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: ** Compaction Stats [default] ** 2024-07-15T06:37:16.110 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-07-15T06:37:16.110 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-07-15T06:37:16.110 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: L0 2/0 80.14 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 92.7 0.00 0.00 1 0.001 0 0 2024-07-15T06:37:16.110 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: Sum 2/0 80.14 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 92.7 0.00 0.00 1 0.001 0 0 2024-07-15T06:37:16.110 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-07-15T06:37:16.110 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: ** Compaction Stats [default] ** 2024-07-15T06:37:16.110 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-07-15T06:37:16.110 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-07-15T06:37:16.110 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 92.7 0.00 0.00 1 0.001 0 0 2024-07-15T06:37:16.110 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: Uptime(secs): 0.0 total, 0.0 interval 2024-07-15T06:37:16.110 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: Flush(GB): cumulative 0.000, interval 0.000 2024-07-15T06:37:16.111 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: AddFile(GB): cumulative 0.000, interval 0.000 2024-07-15T06:37:16.111 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: AddFile(Total Files): cumulative 0, interval 0 2024-07-15T06:37:16.111 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: AddFile(L0 Files): cumulative 0, interval 0 2024-07-15T06:37:16.111 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: AddFile(Keys): cumulative 0, interval 0 2024-07-15T06:37:16.111 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: Cumulative compaction: 0.00 GB write, 13.93 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-07-15T06:37:16.111 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-07-15T06:37:16.111 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-07-15T06:37:16.111 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: ** File Read Latency Histogram By Level [default] ** 2024-07-15T06:37:16.111 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.872+0000 7fb0c3043900 0 starting mon.a rank 0 at public addrs [v2:172.21.15.89:3300/0,v1:172.21.15.89:6789/0] at bind addrs [v2:172.21.15.89:3300/0,v1:172.21.15.89:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 75ba02bc-4274-11ef-bcad-c7b262605968 2024-07-15T06:37:16.111 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.872+0000 7fb0c3043900 1 mon.a@-1(???) e1 preinit fsid 75ba02bc-4274-11ef-bcad-c7b262605968 2024-07-15T06:37:16.111 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.872+0000 7fb0c3043900 0 mon.a@-1(???).mds e1 new map 2024-07-15T06:37:16.111 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.872+0000 7fb0c3043900 0 mon.a@-1(???).mds e1 print_map 2024-07-15T06:37:16.111 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: e1 2024-07-15T06:37:16.111 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: enable_multiple, ever_enabled_multiple: 1,1 2024-07-15T06:37:16.112 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-07-15T06:37:16.112 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: legacy client fscid: -1 2024-07-15T06:37:16.112 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: 2024-07-15T06:37:16.112 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: No filesystems configured 2024-07-15T06:37:16.112 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.872+0000 7fb0c3043900 0 mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-07-15T06:37:16.112 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.872+0000 7fb0c3043900 0 mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-07-15T06:37:16.112 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.872+0000 7fb0c3043900 0 mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-07-15T06:37:16.112 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.872+0000 7fb0c3043900 0 mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-07-15T06:37:16.112 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: debug 2024-07-15T06:37:15.872+0000 7fb0c3043900 1 mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-07-15T06:37:16.112 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: cluster 2024-07-15T06:37:15.907929+0000 mon.a (mon.0) 1 : cluster [INF] mon.a is new leader, mons a in quorum (ranks 0) 2024-07-15T06:37:16.112 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: cluster 2024-07-15T06:37:15.908021+0000 mon.a (mon.0) 2 : cluster [DBG] monmap e1: 1 mons at {a=[v2:172.21.15.89:3300/0,v1:172.21.15.89:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-07-15T06:37:16.112 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: cluster 2024-07-15T06:37:15.908673+0000 mon.a (mon.0) 3 : cluster [DBG] fsmap 2024-07-15T06:37:16.112 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: cluster 2024-07-15T06:37:15.908713+0000 mon.a (mon.0) 4 : cluster [DBG] osdmap e1: 0 total, 0 up, 0 in 2024-07-15T06:37:16.112 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:15 smithi089 bash[21793]: cluster 2024-07-15T06:37:15.909347+0000 mon.a (mon.0) 5 : cluster [DBG] mgrmap e1: no daemons active 2024-07-15T06:37:16.617 INFO:teuthology.orchestra.run.smithi089.stdout:Wrote config to /etc/ceph/ceph.conf 2024-07-15T06:37:16.618 INFO:teuthology.orchestra.run.smithi089.stdout:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-07-15T06:37:16.618 INFO:teuthology.orchestra.run.smithi089.stdout:Creating mgr... 2024-07-15T06:37:16.618 INFO:teuthology.orchestra.run.smithi089.stdout:Verifying port 9283 ... 2024-07-15T06:37:16.858 INFO:teuthology.orchestra.run.smithi089.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-75ba02bc-4274-11ef-bcad-c7b262605968@mgr.y 2024-07-15T06:37:16.858 INFO:teuthology.orchestra.run.smithi089.stdout:systemctl: stderr Failed to reset failed state of unit ceph-75ba02bc-4274-11ef-bcad-c7b262605968@mgr.y.service: Unit ceph-75ba02bc-4274-11ef-bcad-c7b262605968@mgr.y.service not loaded. 2024-07-15T06:37:16.985 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:16 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:37:17.071 INFO:teuthology.orchestra.run.smithi089.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968.target.wants/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@mgr.y.service -> /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service. 2024-07-15T06:37:17.080 INFO:teuthology.orchestra.run.smithi089.stdout:firewalld does not appear to be present 2024-07-15T06:37:17.080 INFO:teuthology.orchestra.run.smithi089.stdout:Not possible to enable service . firewalld.service is not available 2024-07-15T06:37:17.080 INFO:teuthology.orchestra.run.smithi089.stdout:firewalld does not appear to be present 2024-07-15T06:37:17.080 INFO:teuthology.orchestra.run.smithi089.stdout:Not possible to open ports <[9283]>. firewalld.service is not available 2024-07-15T06:37:17.080 INFO:teuthology.orchestra.run.smithi089.stdout:Waiting for mgr to start... 2024-07-15T06:37:17.080 INFO:teuthology.orchestra.run.smithi089.stdout:Waiting for mgr... 2024-07-15T06:37:17.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:16 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:37:17.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:17 smithi089 bash[21793]: audit 2024-07-15T06:37:16.078102+0000 mon.a (mon.0) 6 : audit [INF] from='client.? 172.21.15.89:0/698921605' entity='client.admin' 2024-07-15T06:37:18.872 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:37:18 smithi089 bash[22045]: debug 2024-07-15T06:37:18.716+0000 7f44fb667ec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-07-15T06:37:19.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:18 smithi089 bash[21793]: audit 2024-07-15T06:37:18.868603+0000 mon.a (mon.0) 7 : audit [DBG] from='client.? 172.21.15.89:0/3356274558' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-07-15T06:37:19.441 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout 2024-07-15T06:37:19.441 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout { 2024-07-15T06:37:19.441 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "fsid": "75ba02bc-4274-11ef-bcad-c7b262605968", 2024-07-15T06:37:19.442 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "health": { 2024-07-15T06:37:19.442 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-07-15T06:37:19.442 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-07-15T06:37:19.442 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-07-15T06:37:19.442 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout }, 2024-07-15T06:37:19.442 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-07-15T06:37:19.442 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-07-15T06:37:19.442 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout 0 2024-07-15T06:37:19.442 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout ], 2024-07-15T06:37:19.442 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-07-15T06:37:19.442 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "a" 2024-07-15T06:37:19.442 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout ], 2024-07-15T06:37:19.443 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "quorum_age": 2, 2024-07-15T06:37:19.443 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "monmap": { 2024-07-15T06:37:19.443 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-07-15T06:37:19.443 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2024-07-15T06:37:19.443 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-07-15T06:37:19.443 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout }, 2024-07-15T06:37:19.443 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-07-15T06:37:19.443 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-07-15T06:37:19.443 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-07-15T06:37:19.443 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-07-15T06:37:19.443 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-07-15T06:37:19.443 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-07-15T06:37:19.443 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-07-15T06:37:19.444 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-07-15T06:37:19.444 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout }, 2024-07-15T06:37:19.444 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-07-15T06:37:19.444 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-07-15T06:37:19.444 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-07-15T06:37:19.444 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-07-15T06:37:19.444 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-07-15T06:37:19.444 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-07-15T06:37:19.444 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-07-15T06:37:19.444 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-07-15T06:37:19.444 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-07-15T06:37:19.444 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout }, 2024-07-15T06:37:19.444 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-07-15T06:37:19.445 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-07-15T06:37:19.445 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-07-15T06:37:19.445 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-07-15T06:37:19.445 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout }, 2024-07-15T06:37:19.445 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-07-15T06:37:19.445 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "available": false, 2024-07-15T06:37:19.445 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-07-15T06:37:19.445 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "modules": [ 2024-07-15T06:37:19.445 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "iostat", 2024-07-15T06:37:19.445 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "nfs", 2024-07-15T06:37:19.445 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "restful" 2024-07-15T06:37:19.445 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout ], 2024-07-15T06:37:19.445 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "services": {} 2024-07-15T06:37:19.445 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout }, 2024-07-15T06:37:19.446 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-07-15T06:37:19.446 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-07-15T06:37:19.446 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "modified": "2024-07-15T06:37:09.635525+0000", 2024-07-15T06:37:19.446 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "services": {} 2024-07-15T06:37:19.446 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout }, 2024-07-15T06:37:19.446 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-07-15T06:37:19.446 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout } 2024-07-15T06:37:19.446 INFO:teuthology.orchestra.run.smithi089.stdout:mgr not available, waiting (1/15)... 2024-07-15T06:37:20.372 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:37:19 smithi089 bash[22045]: debug 2024-07-15T06:37:19.928+0000 7f44fb667ec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-07-15T06:37:20.372 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:37:19 smithi089 bash[22045]: debug 2024-07-15T06:37:19.992+0000 7f44fb667ec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-07-15T06:37:20.372 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:37:20 smithi089 bash[22045]: debug 2024-07-15T06:37:20.144+0000 7f44fb667ec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-07-15T06:37:21.076 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:37:20 smithi089 bash[22045]: debug 2024-07-15T06:37:20.780+0000 7f44fb667ec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-07-15T06:37:21.076 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:37:20 smithi089 bash[22045]: debug 2024-07-15T06:37:20.848+0000 7f44fb667ec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-07-15T06:37:21.077 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:37:20 smithi089 bash[22045]: debug 2024-07-15T06:37:20.916+0000 7f44fb667ec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-07-15T06:37:21.372 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:37:21 smithi089 bash[22045]: debug 2024-07-15T06:37:21.076+0000 7f44fb667ec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-07-15T06:37:21.372 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:37:21 smithi089 bash[22045]: debug 2024-07-15T06:37:21.148+0000 7f44fb667ec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-07-15T06:37:21.372 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:37:21 smithi089 bash[22045]: debug 2024-07-15T06:37:21.216+0000 7f44fb667ec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-07-15T06:37:21.762 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:37:21 smithi089 bash[22045]: debug 2024-07-15T06:37:21.448+0000 7f44fb667ec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-07-15T06:37:21.763 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:37:21 smithi089 bash[22045]: debug 2024-07-15T06:37:21.516+0000 7f44fb667ec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-07-15T06:37:21.763 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:37:21 smithi089 bash[22045]: debug 2024-07-15T06:37:21.684+0000 7f44fb667ec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-07-15T06:37:22.122 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:37:21 smithi089 bash[22045]: debug 2024-07-15T06:37:21.760+0000 7f44fb667ec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-07-15T06:37:22.122 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:37:22 smithi089 bash[22045]: debug 2024-07-15T06:37:22.000+0000 7f44fb667ec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-07-15T06:37:22.452 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:37:22 smithi089 bash[22045]: debug 2024-07-15T06:37:22.160+0000 7f44fb667ec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-07-15T06:37:22.452 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:37:22 smithi089 bash[22045]: debug 2024-07-15T06:37:22.240+0000 7f44fb667ec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-07-15T06:37:22.453 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:37:22 smithi089 bash[22045]: debug 2024-07-15T06:37:22.396+0000 7f44fb667ec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-07-15T06:37:23.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:22 smithi089 bash[21793]: audit 2024-07-15T06:37:22.726784+0000 mon.a (mon.0) 8 : audit [DBG] from='client.? 172.21.15.89:0/3481510302' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-07-15T06:37:23.289 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout 2024-07-15T06:37:23.289 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout { 2024-07-15T06:37:23.289 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "fsid": "75ba02bc-4274-11ef-bcad-c7b262605968", 2024-07-15T06:37:23.289 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "health": { 2024-07-15T06:37:23.289 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-07-15T06:37:23.290 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-07-15T06:37:23.290 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-07-15T06:37:23.290 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout }, 2024-07-15T06:37:23.290 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-07-15T06:37:23.290 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-07-15T06:37:23.290 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout 0 2024-07-15T06:37:23.290 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout ], 2024-07-15T06:37:23.291 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-07-15T06:37:23.291 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "a" 2024-07-15T06:37:23.291 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout ], 2024-07-15T06:37:23.291 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "quorum_age": 6, 2024-07-15T06:37:23.291 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "monmap": { 2024-07-15T06:37:23.291 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-07-15T06:37:23.291 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2024-07-15T06:37:23.291 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-07-15T06:37:23.291 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout }, 2024-07-15T06:37:23.291 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-07-15T06:37:23.291 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-07-15T06:37:23.291 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-07-15T06:37:23.292 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-07-15T06:37:23.292 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-07-15T06:37:23.292 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-07-15T06:37:23.292 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-07-15T06:37:23.292 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-07-15T06:37:23.295 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout }, 2024-07-15T06:37:23.295 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-07-15T06:37:23.295 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-07-15T06:37:23.296 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-07-15T06:37:23.296 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-07-15T06:37:23.296 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-07-15T06:37:23.296 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-07-15T06:37:23.296 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-07-15T06:37:23.296 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-07-15T06:37:23.296 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-07-15T06:37:23.296 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout }, 2024-07-15T06:37:23.296 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-07-15T06:37:23.296 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-07-15T06:37:23.296 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-07-15T06:37:23.296 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-07-15T06:37:23.296 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout }, 2024-07-15T06:37:23.297 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-07-15T06:37:23.297 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "available": false, 2024-07-15T06:37:23.297 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-07-15T06:37:23.297 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "modules": [ 2024-07-15T06:37:23.297 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "iostat", 2024-07-15T06:37:23.297 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "nfs", 2024-07-15T06:37:23.297 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "restful" 2024-07-15T06:37:23.297 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout ], 2024-07-15T06:37:23.297 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "services": {} 2024-07-15T06:37:23.297 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout }, 2024-07-15T06:37:23.297 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-07-15T06:37:23.297 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-07-15T06:37:23.297 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "modified": "2024-07-15T06:37:09.635525+0000", 2024-07-15T06:37:23.298 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "services": {} 2024-07-15T06:37:23.298 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout }, 2024-07-15T06:37:23.298 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-07-15T06:37:23.298 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout } 2024-07-15T06:37:23.298 INFO:teuthology.orchestra.run.smithi089.stdout:mgr not available, waiting (2/15)... 2024-07-15T06:37:23.481 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:37:23 smithi089 bash[22045]: debug 2024-07-15T06:37:23.200+0000 7f44fb667ec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-07-15T06:37:23.481 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:37:23 smithi089 bash[22045]: debug 2024-07-15T06:37:23.276+0000 7f44fb667ec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-07-15T06:37:23.868 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:37:23 smithi089 bash[22045]: debug 2024-07-15T06:37:23.480+0000 7f44fb667ec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-07-15T06:37:23.869 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:37:23 smithi089 bash[22045]: debug 2024-07-15T06:37:23.552+0000 7f44fb667ec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-07-15T06:37:24.122 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:37:23 smithi089 bash[22045]: debug 2024-07-15T06:37:23.868+0000 7f44fb667ec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-07-15T06:37:24.122 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:37:23 smithi089 bash[22045]: debug 2024-07-15T06:37:23.944+0000 7f44fb667ec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-07-15T06:37:24.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:24 smithi089 bash[21793]: cluster 2024-07-15T06:37:23.969418+0000 mon.a (mon.0) 9 : cluster [INF] Activating manager daemon y 2024-07-15T06:37:24.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:24 smithi089 bash[21793]: cluster 2024-07-15T06:37:23.972570+0000 mon.a (mon.0) 10 : cluster [DBG] mgrmap e2: y(active, starting, since 0.00333825s) 2024-07-15T06:37:24.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:24 smithi089 bash[21793]: audit 2024-07-15T06:37:23.975651+0000 mon.a (mon.0) 11 : audit [DBG] from='mgr.14100 172.21.15.89:0/2456426930' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-07-15T06:37:24.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:24 smithi089 bash[21793]: audit 2024-07-15T06:37:23.975829+0000 mon.a (mon.0) 12 : audit [DBG] from='mgr.14100 172.21.15.89:0/2456426930' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-07-15T06:37:24.123 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:24 smithi089 bash[21793]: audit 2024-07-15T06:37:23.975943+0000 mon.a (mon.0) 13 : audit [DBG] from='mgr.14100 172.21.15.89:0/2456426930' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-07-15T06:37:24.123 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:24 smithi089 bash[21793]: audit 2024-07-15T06:37:23.977043+0000 mon.a (mon.0) 14 : audit [DBG] from='mgr.14100 172.21.15.89:0/2456426930' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-07-15T06:37:24.123 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:24 smithi089 bash[21793]: audit 2024-07-15T06:37:23.977517+0000 mon.a (mon.0) 15 : audit [DBG] from='mgr.14100 172.21.15.89:0/2456426930' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-07-15T06:37:24.123 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:24 smithi089 bash[21793]: cluster 2024-07-15T06:37:24.001246+0000 mon.a (mon.0) 16 : cluster [INF] Manager daemon y is now available 2024-07-15T06:37:24.123 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:24 smithi089 bash[21793]: audit 2024-07-15T06:37:24.013535+0000 mon.a (mon.0) 17 : audit [INF] from='mgr.14100 172.21.15.89:0/2456426930' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-07-15T06:37:24.123 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:24 smithi089 bash[21793]: audit 2024-07-15T06:37:24.016192+0000 mon.a (mon.0) 18 : audit [INF] from='mgr.14100 172.21.15.89:0/2456426930' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-07-15T06:37:24.123 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:24 smithi089 bash[21793]: audit 2024-07-15T06:37:24.018378+0000 mon.a (mon.0) 19 : audit [INF] from='mgr.14100 172.21.15.89:0/2456426930' entity='mgr.y' 2024-07-15T06:37:24.123 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:24 smithi089 bash[21793]: audit 2024-07-15T06:37:24.020256+0000 mon.a (mon.0) 20 : audit [INF] from='mgr.14100 172.21.15.89:0/2456426930' entity='mgr.y' 2024-07-15T06:37:25.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:25 smithi089 bash[21793]: audit 2024-07-15T06:37:24.024066+0000 mon.a (mon.0) 21 : audit [INF] from='mgr.14100 172.21.15.89:0/2456426930' entity='mgr.y' 2024-07-15T06:37:25.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:25 smithi089 bash[21793]: cluster 2024-07-15T06:37:24.983366+0000 mon.a (mon.0) 22 : cluster [DBG] mgrmap e3: y(active, since 1.01413s) 2024-07-15T06:37:27.363 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout 2024-07-15T06:37:27.363 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout { 2024-07-15T06:37:27.363 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "fsid": "75ba02bc-4274-11ef-bcad-c7b262605968", 2024-07-15T06:37:27.363 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "health": { 2024-07-15T06:37:27.363 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-07-15T06:37:27.363 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-07-15T06:37:27.363 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-07-15T06:37:27.364 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout }, 2024-07-15T06:37:27.364 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-07-15T06:37:27.364 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-07-15T06:37:27.364 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout 0 2024-07-15T06:37:27.364 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout ], 2024-07-15T06:37:27.364 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-07-15T06:37:27.364 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "a" 2024-07-15T06:37:27.364 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout ], 2024-07-15T06:37:27.364 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "quorum_age": 10, 2024-07-15T06:37:27.364 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "monmap": { 2024-07-15T06:37:27.364 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-07-15T06:37:27.364 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2024-07-15T06:37:27.364 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-07-15T06:37:27.365 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout }, 2024-07-15T06:37:27.365 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-07-15T06:37:27.365 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-07-15T06:37:27.365 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-07-15T06:37:27.365 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-07-15T06:37:27.365 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-07-15T06:37:27.365 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-07-15T06:37:27.365 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-07-15T06:37:27.365 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-07-15T06:37:27.368 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout }, 2024-07-15T06:37:27.368 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-07-15T06:37:27.368 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-07-15T06:37:27.369 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-07-15T06:37:27.369 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-07-15T06:37:27.369 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-07-15T06:37:27.369 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-07-15T06:37:27.369 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-07-15T06:37:27.369 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-07-15T06:37:27.369 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-07-15T06:37:27.369 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout }, 2024-07-15T06:37:27.369 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-07-15T06:37:27.369 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-07-15T06:37:27.369 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-07-15T06:37:27.369 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-07-15T06:37:27.369 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout }, 2024-07-15T06:37:27.369 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-07-15T06:37:27.370 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "available": true, 2024-07-15T06:37:27.370 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-07-15T06:37:27.370 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "modules": [ 2024-07-15T06:37:27.370 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "iostat", 2024-07-15T06:37:27.370 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "nfs", 2024-07-15T06:37:27.370 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "restful" 2024-07-15T06:37:27.370 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout ], 2024-07-15T06:37:27.370 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "services": {} 2024-07-15T06:37:27.370 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout }, 2024-07-15T06:37:27.370 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-07-15T06:37:27.370 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-07-15T06:37:27.370 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "modified": "2024-07-15T06:37:09.635525+0000", 2024-07-15T06:37:27.370 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "services": {} 2024-07-15T06:37:27.371 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout }, 2024-07-15T06:37:27.371 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-07-15T06:37:27.371 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout } 2024-07-15T06:37:27.371 INFO:teuthology.orchestra.run.smithi089.stdout:mgr is available 2024-07-15T06:37:27.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:27 smithi089 bash[21793]: cluster 2024-07-15T06:37:26.033692+0000 mon.a (mon.0) 23 : cluster [DBG] mgrmap e4: y(active, since 2s) 2024-07-15T06:37:27.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:27 smithi089 bash[21793]: audit 2024-07-15T06:37:26.889289+0000 mon.a (mon.0) 24 : audit [DBG] from='client.? 172.21.15.89:0/876424233' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-07-15T06:37:29.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:28 smithi089 bash[21793]: audit 2024-07-15T06:37:28.681660+0000 mon.a (mon.0) 25 : audit [INF] from='client.? 172.21.15.89:0/284812335' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-07-15T06:37:29.159 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout 2024-07-15T06:37:29.160 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout [global] 2024-07-15T06:37:29.160 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout fsid = 75ba02bc-4274-11ef-bcad-c7b262605968 2024-07-15T06:37:29.160 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2024-07-15T06:37:29.160 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2024-07-15T06:37:29.160 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2024-07-15T06:37:29.160 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2024-07-15T06:37:29.160 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout 2024-07-15T06:37:29.160 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout [mgr] 2024-07-15T06:37:29.160 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2024-07-15T06:37:29.160 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout 2024-07-15T06:37:29.160 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout [osd] 2024-07-15T06:37:29.160 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2024-07-15T06:37:29.161 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2024-07-15T06:37:29.161 INFO:teuthology.orchestra.run.smithi089.stdout:Enabling cephadm module... 2024-07-15T06:37:30.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:30 smithi089 bash[21793]: audit 2024-07-15T06:37:30.562018+0000 mon.a (mon.0) 26 : audit [INF] from='client.? 172.21.15.89:0/3998031561' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-07-15T06:37:30.872 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:37:30 smithi089 bash[22045]: ignoring --setuser ceph since I am not root 2024-07-15T06:37:30.873 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:37:30 smithi089 bash[22045]: ignoring --setgroup ceph since I am not root 2024-07-15T06:37:30.873 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:37:30 smithi089 bash[22045]: debug 2024-07-15T06:37:30.751+0000 7f7cdbc0eec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-07-15T06:37:31.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:31 smithi089 bash[21793]: audit 2024-07-15T06:37:30.617381+0000 mon.a (mon.0) 27 : audit [INF] from='client.? 172.21.15.89:0/3998031561' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-07-15T06:37:31.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:31 smithi089 bash[21793]: cluster 2024-07-15T06:37:30.617419+0000 mon.a (mon.0) 28 : cluster [DBG] mgrmap e5: y(active, since 6s) 2024-07-15T06:37:32.372 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:37:31 smithi089 bash[22045]: debug 2024-07-15T06:37:31.951+0000 7f7cdbc0eec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-07-15T06:37:32.372 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:37:32 smithi089 bash[22045]: debug 2024-07-15T06:37:32.019+0000 7f7cdbc0eec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-07-15T06:37:32.372 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:37:32 smithi089 bash[22045]: debug 2024-07-15T06:37:32.179+0000 7f7cdbc0eec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-07-15T06:37:32.826 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:32 smithi089 bash[21793]: audit 2024-07-15T06:37:32.548977+0000 mon.a (mon.0) 29 : audit [DBG] from='client.? 172.21.15.89:0/4169092585' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-07-15T06:37:33.064 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout { 2024-07-15T06:37:33.064 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "epoch": 5, 2024-07-15T06:37:33.064 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "available": true, 2024-07-15T06:37:33.065 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "active_name": "y", 2024-07-15T06:37:33.065 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "num_standby": 0 2024-07-15T06:37:33.065 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout } 2024-07-15T06:37:33.065 INFO:teuthology.orchestra.run.smithi089.stdout:Waiting for the mgr to restart... 2024-07-15T06:37:33.065 INFO:teuthology.orchestra.run.smithi089.stdout:Waiting for mgr epoch 5... 2024-07-15T06:37:33.122 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:37:32 smithi089 bash[22045]: debug 2024-07-15T06:37:32.823+0000 7f7cdbc0eec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-07-15T06:37:33.122 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:37:32 smithi089 bash[22045]: debug 2024-07-15T06:37:32.891+0000 7f7cdbc0eec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-07-15T06:37:33.122 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:37:32 smithi089 bash[22045]: debug 2024-07-15T06:37:32.959+0000 7f7cdbc0eec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-07-15T06:37:33.500 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:37:33 smithi089 bash[22045]: debug 2024-07-15T06:37:33.123+0000 7f7cdbc0eec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-07-15T06:37:33.501 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:37:33 smithi089 bash[22045]: debug 2024-07-15T06:37:33.199+0000 7f7cdbc0eec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-07-15T06:37:33.501 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:37:33 smithi089 bash[22045]: debug 2024-07-15T06:37:33.267+0000 7f7cdbc0eec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-07-15T06:37:33.815 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:37:33 smithi089 bash[22045]: debug 2024-07-15T06:37:33.495+0000 7f7cdbc0eec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-07-15T06:37:33.815 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:37:33 smithi089 bash[22045]: debug 2024-07-15T06:37:33.563+0000 7f7cdbc0eec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-07-15T06:37:33.815 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:37:33 smithi089 bash[22045]: debug 2024-07-15T06:37:33.735+0000 7f7cdbc0eec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-07-15T06:37:33.815 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:37:33 smithi089 bash[22045]: debug 2024-07-15T06:37:33.811+0000 7f7cdbc0eec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-07-15T06:37:34.122 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:37:34 smithi089 bash[22045]: debug 2024-07-15T06:37:34.055+0000 7f7cdbc0eec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-07-15T06:37:34.622 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:37:34 smithi089 bash[22045]: debug 2024-07-15T06:37:34.223+0000 7f7cdbc0eec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-07-15T06:37:34.623 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:37:34 smithi089 bash[22045]: debug 2024-07-15T06:37:34.299+0000 7f7cdbc0eec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-07-15T06:37:34.623 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:37:34 smithi089 bash[22045]: debug 2024-07-15T06:37:34.451+0000 7f7cdbc0eec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-07-15T06:37:35.493 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:37:35 smithi089 bash[22045]: debug 2024-07-15T06:37:35.215+0000 7f7cdbc0eec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-07-15T06:37:35.493 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:37:35 smithi089 bash[22045]: debug 2024-07-15T06:37:35.291+0000 7f7cdbc0eec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-07-15T06:37:35.872 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:37:35 smithi089 bash[22045]: debug 2024-07-15T06:37:35.487+0000 7f7cdbc0eec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-07-15T06:37:35.873 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:37:35 smithi089 bash[22045]: debug 2024-07-15T06:37:35.563+0000 7f7cdbc0eec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-07-15T06:37:36.372 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:37:35 smithi089 bash[22045]: debug 2024-07-15T06:37:35.875+0000 7f7cdbc0eec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-07-15T06:37:36.373 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:37:35 smithi089 bash[22045]: debug 2024-07-15T06:37:35.955+0000 7f7cdbc0eec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-07-15T06:37:36.373 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:36 smithi089 bash[21793]: cluster 2024-07-15T06:37:35.981259+0000 mon.a (mon.0) 30 : cluster [INF] Active manager daemon y restarted 2024-07-15T06:37:36.373 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:36 smithi089 bash[21793]: cluster 2024-07-15T06:37:35.981972+0000 mon.a (mon.0) 31 : cluster [INF] Activating manager daemon y 2024-07-15T06:37:36.373 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:36 smithi089 bash[21793]: cluster 2024-07-15T06:37:35.989852+0000 mon.a (mon.0) 32 : cluster [DBG] osdmap e2: 0 total, 0 up, 0 in 2024-07-15T06:37:36.373 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:36 smithi089 bash[21793]: cluster 2024-07-15T06:37:35.990076+0000 mon.a (mon.0) 33 : cluster [DBG] mgrmap e6: y(active, starting, since 0.00841194s) 2024-07-15T06:37:36.373 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:36 smithi089 bash[21793]: audit 2024-07-15T06:37:35.991832+0000 mon.a (mon.0) 34 : audit [DBG] from='mgr.14118 172.21.15.89:0/1638228389' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-07-15T06:37:36.373 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:36 smithi089 bash[21793]: audit 2024-07-15T06:37:35.993830+0000 mon.a (mon.0) 35 : audit [DBG] from='mgr.14118 172.21.15.89:0/1638228389' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-07-15T06:37:36.374 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:36 smithi089 bash[21793]: audit 2024-07-15T06:37:35.995008+0000 mon.a (mon.0) 36 : audit [DBG] from='mgr.14118 172.21.15.89:0/1638228389' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-07-15T06:37:36.374 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:36 smithi089 bash[21793]: audit 2024-07-15T06:37:35.995452+0000 mon.a (mon.0) 37 : audit [DBG] from='mgr.14118 172.21.15.89:0/1638228389' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-07-15T06:37:36.374 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:36 smithi089 bash[21793]: audit 2024-07-15T06:37:35.995664+0000 mon.a (mon.0) 38 : audit [DBG] from='mgr.14118 172.21.15.89:0/1638228389' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-07-15T06:37:36.374 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:36 smithi089 bash[21793]: cluster 2024-07-15T06:37:36.002737+0000 mon.a (mon.0) 39 : cluster [INF] Manager daemon y is now available 2024-07-15T06:37:36.374 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:36 smithi089 bash[21793]: audit 2024-07-15T06:37:36.022488+0000 mon.a (mon.0) 40 : audit [INF] from='mgr.14118 172.21.15.89:0/1638228389' entity='mgr.y' 2024-07-15T06:37:36.374 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:36 smithi089 bash[21793]: audit 2024-07-15T06:37:36.028921+0000 mon.a (mon.0) 41 : audit [INF] from='mgr.14118 172.21.15.89:0/1638228389' entity='mgr.y' 2024-07-15T06:37:36.374 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:36 smithi089 bash[21793]: audit 2024-07-15T06:37:36.031885+0000 mon.a (mon.0) 42 : audit [DBG] from='mgr.14118 172.21.15.89:0/1638228389' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:37:37.373 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:37 smithi089 bash[21793]: cephadm 2024-07-15T06:37:36.016394+0000 mgr.y (mgr.14118) 1 : cephadm [INF] Found migration_current of "None". Setting to last migration. 2024-07-15T06:37:37.373 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:37 smithi089 bash[21793]: audit 2024-07-15T06:37:36.034116+0000 mon.a (mon.0) 43 : audit [DBG] from='mgr.14118 172.21.15.89:0/1638228389' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:37:37.373 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:37 smithi089 bash[21793]: audit 2024-07-15T06:37:36.036094+0000 mon.a (mon.0) 44 : audit [DBG] from='mgr.14118 172.21.15.89:0/1638228389' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:37:37.373 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:37 smithi089 bash[21793]: audit 2024-07-15T06:37:36.045182+0000 mon.a (mon.0) 45 : audit [INF] from='mgr.14118 172.21.15.89:0/1638228389' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-07-15T06:37:37.373 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:37 smithi089 bash[21793]: audit 2024-07-15T06:37:36.047423+0000 mon.a (mon.0) 46 : audit [INF] from='mgr.14118 172.21.15.89:0/1638228389' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-07-15T06:37:37.373 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:37 smithi089 bash[21793]: cluster 2024-07-15T06:37:36.993964+0000 mon.a (mon.0) 47 : cluster [DBG] mgrmap e7: y(active, since 1.01229s) 2024-07-15T06:37:37.576 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout { 2024-07-15T06:37:37.576 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 7, 2024-07-15T06:37:37.576 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "initialized": true 2024-07-15T06:37:37.577 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout } 2024-07-15T06:37:37.577 INFO:teuthology.orchestra.run.smithi089.stdout:mgr epoch 5 is available 2024-07-15T06:37:37.577 INFO:teuthology.orchestra.run.smithi089.stdout:Setting orchestrator backend to cephadm... 2024-07-15T06:37:37.873 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:37:37 smithi089 bash[22045]: debug 2024-07-15T06:37:37.567+0000 7f7c8e078640 -1 log_channel(cephadm) log [ERR] : [15/Jul/2024:06:37:37] ENGINE Error in HTTPServer.serve 2024-07-15T06:37:37.873 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:37:37 smithi089 bash[22045]: Traceback (most recent call last): 2024-07-15T06:37:37.873 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:37:37 smithi089 bash[22045]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-07-15T06:37:37.873 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:37:37 smithi089 bash[22045]: self._connections.run(self.expiration_interval) 2024-07-15T06:37:37.874 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:37:37 smithi089 bash[22045]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-07-15T06:37:37.874 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:37:37 smithi089 bash[22045]: self._run(expiration_interval) 2024-07-15T06:37:37.874 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:37:37 smithi089 bash[22045]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-07-15T06:37:37.874 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:37:37 smithi089 bash[22045]: new_conn = self._from_server_socket(self.server.socket) 2024-07-15T06:37:37.874 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:37:37 smithi089 bash[22045]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-07-15T06:37:37.874 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:37:37 smithi089 bash[22045]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-07-15T06:37:37.874 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:37:37 smithi089 bash[22045]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-07-15T06:37:37.874 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:37:37 smithi089 bash[22045]: s = self.context.wrap_socket( 2024-07-15T06:37:37.874 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:37:37 smithi089 bash[22045]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-07-15T06:37:37.874 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:37:37 smithi089 bash[22045]: return self.sslsocket_class._create( 2024-07-15T06:37:37.874 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:37:37 smithi089 bash[22045]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-07-15T06:37:37.874 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:37:37 smithi089 bash[22045]: self.do_handshake() 2024-07-15T06:37:37.874 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:37:37 smithi089 bash[22045]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-07-15T06:37:37.875 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:37:37 smithi089 bash[22045]: self._sslobj.do_handshake() 2024-07-15T06:37:37.875 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:37:37 smithi089 bash[22045]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-07-15T06:37:38.873 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:38 smithi089 bash[21793]: audit 2024-07-15T06:37:36.992274+0000 mgr.y (mgr.14118) 2 : audit [DBG] from='client.14122 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-07-15T06:37:38.874 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:38 smithi089 bash[21793]: audit 2024-07-15T06:37:37.003779+0000 mgr.y (mgr.14118) 3 : audit [DBG] from='client.14122 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-07-15T06:37:38.874 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:38 smithi089 bash[21793]: cephadm 2024-07-15T06:37:37.451742+0000 mgr.y (mgr.14118) 4 : cephadm [INF] [15/Jul/2024:06:37:37] ENGINE Bus STARTING 2024-07-15T06:37:38.874 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:38 smithi089 bash[21793]: cephadm 2024-07-15T06:37:37.572667+0000 mgr.y (mgr.14118) 5 : cephadm [INF] [15/Jul/2024:06:37:37] ENGINE Serving on https://172.21.15.89:7150 2024-07-15T06:37:38.874 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:38 smithi089 bash[21793]: cephadm 2024-07-15T06:37:37.572789+0000 mgr.y (mgr.14118) 6 : cephadm [INF] [15/Jul/2024:06:37:37] ENGINE Bus STARTED 2024-07-15T06:37:38.874 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:38 smithi089 bash[21793]: cephadm 2024-07-15T06:37:37.574661+0000 mgr.y (mgr.14118) 7 : cephadm [ERR] [15/Jul/2024:06:37:37] ENGINE Error in HTTPServer.serve 2024-07-15T06:37:38.874 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:38 smithi089 bash[21793]: Traceback (most recent call last): 2024-07-15T06:37:38.874 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:38 smithi089 bash[21793]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-07-15T06:37:38.874 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:38 smithi089 bash[21793]: self._connections.run(self.expiration_interval) 2024-07-15T06:37:38.874 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:38 smithi089 bash[21793]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-07-15T06:37:38.874 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:38 smithi089 bash[21793]: self._run(expiration_interval) 2024-07-15T06:37:38.874 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:38 smithi089 bash[21793]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-07-15T06:37:38.875 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:38 smithi089 bash[21793]: new_conn = self._from_server_socket(self.server.socket) 2024-07-15T06:37:38.875 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:38 smithi089 bash[21793]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-07-15T06:37:38.875 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:38 smithi089 bash[21793]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-07-15T06:37:38.875 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:38 smithi089 bash[21793]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-07-15T06:37:38.875 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:38 smithi089 bash[21793]: s = self.context.wrap_socket( 2024-07-15T06:37:38.875 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:38 smithi089 bash[21793]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-07-15T06:37:38.875 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:38 smithi089 bash[21793]: return self.sslsocket_class._create( 2024-07-15T06:37:38.875 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:38 smithi089 bash[21793]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-07-15T06:37:38.875 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:38 smithi089 bash[21793]: self.do_handshake() 2024-07-15T06:37:38.875 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:38 smithi089 bash[21793]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-07-15T06:37:38.875 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:38 smithi089 bash[21793]: self._sslobj.do_handshake() 2024-07-15T06:37:38.875 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:38 smithi089 bash[21793]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-07-15T06:37:38.875 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:38 smithi089 bash[21793]: audit 2024-07-15T06:37:37.578318+0000 mon.a (mon.0) 48 : audit [INF] from='mgr.14118 172.21.15.89:0/1638228389' entity='mgr.y' 2024-07-15T06:37:38.876 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:38 smithi089 bash[21793]: audit 2024-07-15T06:37:37.579742+0000 mon.a (mon.0) 49 : audit [DBG] from='mgr.14118 172.21.15.89:0/1638228389' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:37:39.873 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:39 smithi089 bash[21793]: cluster 2024-07-15T06:37:38.581270+0000 mon.a (mon.0) 50 : cluster [DBG] mgrmap e8: y(active, since 2s) 2024-07-15T06:37:39.873 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:39 smithi089 bash[21793]: audit 2024-07-15T06:37:38.984745+0000 mon.a (mon.0) 51 : audit [INF] from='mgr.14118 172.21.15.89:0/1638228389' entity='mgr.y' 2024-07-15T06:37:39.873 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:39 smithi089 bash[21793]: audit 2024-07-15T06:37:38.998121+0000 mon.a (mon.0) 52 : audit [DBG] from='mgr.14118 172.21.15.89:0/1638228389' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:37:40.873 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:40 smithi089 bash[21793]: audit 2024-07-15T06:37:38.977602+0000 mgr.y (mgr.14118) 8 : audit [DBG] from='client.14130 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T06:37:41.485 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout value unchanged 2024-07-15T06:37:41.485 INFO:teuthology.orchestra.run.smithi089.stdout:Generating ssh key... 2024-07-15T06:37:41.873 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:41 smithi089 bash[21793]: audit 2024-07-15T06:37:40.955373+0000 mgr.y (mgr.14118) 9 : audit [DBG] from='client.14132 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T06:37:43.373 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:37:42 smithi089 bash[22045]: Generating public/private rsa key pair. 2024-07-15T06:37:43.374 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:37:42 smithi089 bash[22045]: Your identification has been saved in /tmp/tmp3mrtfgwv/key 2024-07-15T06:37:43.374 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:37:42 smithi089 bash[22045]: Your public key has been saved in /tmp/tmp3mrtfgwv/key.pub 2024-07-15T06:37:43.374 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:37:42 smithi089 bash[22045]: The key fingerprint is: 2024-07-15T06:37:43.374 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:37:42 smithi089 bash[22045]: SHA256:6XqlH4vDzvivdc7sqj+G3/KM8Qfv2SD8IoMSYvv5Fqg ceph-75ba02bc-4274-11ef-bcad-c7b262605968 2024-07-15T06:37:43.374 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:37:42 smithi089 bash[22045]: The key's randomart image is: 2024-07-15T06:37:43.374 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:37:42 smithi089 bash[22045]: +---[RSA 3072]----+ 2024-07-15T06:37:43.374 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:37:42 smithi089 bash[22045]: | | 2024-07-15T06:37:43.374 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:37:42 smithi089 bash[22045]: | | 2024-07-15T06:37:43.374 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:37:42 smithi089 bash[22045]: | | 2024-07-15T06:37:43.374 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:37:42 smithi089 bash[22045]: | . | 2024-07-15T06:37:43.374 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:37:42 smithi089 bash[22045]: | .S | 2024-07-15T06:37:43.375 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:37:42 smithi089 bash[22045]: | o o.. ... | 2024-07-15T06:37:43.375 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:37:42 smithi089 bash[22045]: | . + o.*+ +o. | 2024-07-15T06:37:43.375 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:37:42 smithi089 bash[22045]: | E .=Oo*&.oo+ | 2024-07-15T06:37:43.375 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:37:42 smithi089 bash[22045]: | .=BOXOOX++ .| 2024-07-15T06:37:43.375 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:37:42 smithi089 bash[22045]: +----[SHA256]-----+ 2024-07-15T06:37:44.367 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:43 smithi089 bash[21793]: audit 2024-07-15T06:37:42.818116+0000 mgr.y (mgr.14118) 10 : audit [DBG] from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T06:37:44.367 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:43 smithi089 bash[21793]: cephadm 2024-07-15T06:37:42.818490+0000 mgr.y (mgr.14118) 11 : cephadm [INF] Generating ssh key... 2024-07-15T06:37:44.367 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:43 smithi089 bash[21793]: audit 2024-07-15T06:37:42.921035+0000 mon.a (mon.0) 53 : audit [INF] from='mgr.14118 172.21.15.89:0/1638228389' entity='mgr.y' 2024-07-15T06:37:44.367 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:43 smithi089 bash[21793]: audit 2024-07-15T06:37:42.925986+0000 mon.a (mon.0) 54 : audit [INF] from='mgr.14118 172.21.15.89:0/1638228389' entity='mgr.y' 2024-07-15T06:37:45.407 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCT4NEbULOb3GWmHnmWxk83Z/qKS+P3BoKW8JNgOvff/ZN904M4hlCC+5mHyx8fpMoaJ8U/+yhaCGOH+yHADlO+1f371CUVoJJbGJ0TF8QR95Qkn0fhWZFhDJoNP6thKcqshDZty4Mlacbccu5ZRGFDv/7yxTNlweDDkUVUil4gdLowCLdiQujGxXzY4skNVhoG+d+ZeMSttrKxgoSOfsZWSAB0+KeobWOTNlZYWmrpfABQneb/PxqKk6QRz0gKFYx7N7R8ToAAKbVZ544n86yST8Wi6NUN9FU9gmk4qAFxOLH5SoM8dpwukVjp8MxL9tPqIBjevpwUT1mdWAi5WczCmti3ncM4xRNutBzfk9T4AuDL/WHkG+n+JeTpuJX1tl9S+ByqVi9IjCEx0qBFbGuCRagKSoTbslyWxNdOFXYZwHLvEgfENiFL6c5omX7q6Itv+p9M/Il8tjCojwEmiIMxzOpJ9DtbufYRO/jHYcMdcrOnMAue//dHypGc+qwRjs8= ceph-75ba02bc-4274-11ef-bcad-c7b262605968 2024-07-15T06:37:45.408 INFO:teuthology.orchestra.run.smithi089.stdout:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-07-15T06:37:45.408 INFO:teuthology.orchestra.run.smithi089.stdout:Adding key to root@localhost authorized_keys... 2024-07-15T06:37:45.408 INFO:teuthology.orchestra.run.smithi089.stdout:Adding host smithi089... 2024-07-15T06:37:45.873 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:45 smithi089 bash[21793]: audit 2024-07-15T06:37:44.861963+0000 mgr.y (mgr.14118) 12 : audit [DBG] from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T06:37:48.009 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:47 smithi089 bash[21793]: audit 2024-07-15T06:37:46.874543+0000 mgr.y (mgr.14118) 13 : audit [DBG] from='client.14138 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi089", "addr": "172.21.15.89", "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T06:37:48.801 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:48 smithi089 bash[21793]: cephadm 2024-07-15T06:37:47.882126+0000 mgr.y (mgr.14118) 14 : cephadm [INF] Deploying cephadm binary to smithi089 2024-07-15T06:37:50.723 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout Added host 'smithi089' with addr '172.21.15.89' 2024-07-15T06:37:50.723 INFO:teuthology.orchestra.run.smithi089.stdout:Deploying unmanaged mon service... 2024-07-15T06:37:51.123 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:50 smithi089 bash[21793]: audit 2024-07-15T06:37:49.963955+0000 mon.a (mon.0) 55 : audit [INF] from='mgr.14118 172.21.15.89:0/1638228389' entity='mgr.y' 2024-07-15T06:37:51.123 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:50 smithi089 bash[21793]: cephadm 2024-07-15T06:37:49.964278+0000 mgr.y (mgr.14118) 15 : cephadm [INF] Added host smithi089 2024-07-15T06:37:51.124 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:50 smithi089 bash[21793]: audit 2024-07-15T06:37:49.964681+0000 mon.a (mon.0) 56 : audit [DBG] from='mgr.14118 172.21.15.89:0/1638228389' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:37:53.115 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:52 smithi089 bash[21793]: audit 2024-07-15T06:37:52.658567+0000 mgr.y (mgr.14118) 16 : audit [DBG] from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T06:37:53.116 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:52 smithi089 bash[21793]: cephadm 2024-07-15T06:37:52.661619+0000 mgr.y (mgr.14118) 17 : cephadm [INF] Saving service mon spec with placement count:5 2024-07-15T06:37:53.116 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:52 smithi089 bash[21793]: audit 2024-07-15T06:37:52.664525+0000 mon.a (mon.0) 57 : audit [INF] from='mgr.14118 172.21.15.89:0/1638228389' entity='mgr.y' 2024-07-15T06:37:53.435 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout Scheduled mon update... 2024-07-15T06:37:53.435 INFO:teuthology.orchestra.run.smithi089.stdout:Deploying unmanaged mgr service... 2024-07-15T06:37:54.623 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:54 smithi089 bash[21793]: audit 2024-07-15T06:37:53.173626+0000 mon.a (mon.0) 58 : audit [INF] from='mgr.14118 172.21.15.89:0/1638228389' entity='mgr.y' 2024-07-15T06:37:54.623 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:54 smithi089 bash[21793]: audit 2024-07-15T06:37:53.444217+0000 mon.a (mon.0) 59 : audit [INF] from='mgr.14118 172.21.15.89:0/1638228389' entity='mgr.y' 2024-07-15T06:37:55.363 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout Scheduled mgr update... 2024-07-15T06:37:56.123 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:55 smithi089 bash[21793]: audit 2024-07-15T06:37:54.782925+0000 mgr.y (mgr.14118) 18 : audit [DBG] from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T06:37:56.123 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:55 smithi089 bash[21793]: cephadm 2024-07-15T06:37:54.783864+0000 mgr.y (mgr.14118) 19 : cephadm [INF] Saving service mgr spec with placement count:2 2024-07-15T06:37:56.123 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:55 smithi089 bash[21793]: audit 2024-07-15T06:37:54.786504+0000 mon.a (mon.0) 60 : audit [INF] from='mgr.14118 172.21.15.89:0/1638228389' entity='mgr.y' 2024-07-15T06:37:57.957 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:57 smithi089 bash[21793]: cluster 2024-07-15T06:37:55.996406+0000 mgr.y (mgr.14118) 20 : cluster [DBG] pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:37:57.957 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:57 smithi089 bash[21793]: audit 2024-07-15T06:37:56.634899+0000 mon.a (mon.0) 61 : audit [INF] from='client.? 172.21.15.89:0/4035427415' entity='client.admin' 2024-07-15T06:37:59.043 INFO:teuthology.orchestra.run.smithi089.stdout:Enabling the dashboard module... 2024-07-15T06:37:59.623 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:59 smithi089 bash[21793]: cluster 2024-07-15T06:37:57.996726+0000 mgr.y (mgr.14118) 21 : cluster [DBG] pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:37:59.624 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:59 smithi089 bash[21793]: audit 2024-07-15T06:37:58.363433+0000 mon.a (mon.0) 62 : audit [INF] from='mgr.14118 172.21.15.89:0/1638228389' entity='mgr.y' 2024-07-15T06:37:59.624 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:59 smithi089 bash[21793]: audit 2024-07-15T06:37:58.478056+0000 mon.a (mon.0) 63 : audit [INF] from='client.? 172.21.15.89:0/77883119' entity='client.admin' 2024-07-15T06:37:59.624 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:37:59 smithi089 bash[21793]: audit 2024-07-15T06:37:58.947135+0000 mon.a (mon.0) 64 : audit [INF] from='mgr.14118 172.21.15.89:0/1638228389' entity='mgr.y' 2024-07-15T06:38:01.627 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:01 smithi089 bash[21793]: cluster 2024-07-15T06:37:59.997025+0000 mgr.y (mgr.14118) 22 : cluster [DBG] pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:38:01.627 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:01 smithi089 bash[21793]: audit 2024-07-15T06:38:00.550079+0000 mon.a (mon.0) 65 : audit [INF] from='client.? 172.21.15.89:0/1203140001' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-07-15T06:38:01.627 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:38:01 smithi089 bash[22045]: ignoring --setuser ceph since I am not root 2024-07-15T06:38:01.628 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:38:01 smithi089 bash[22045]: ignoring --setgroup ceph since I am not root 2024-07-15T06:38:01.628 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:38:01 smithi089 bash[22045]: debug 2024-07-15T06:38:01.525+0000 7ff83263bec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-07-15T06:38:02.623 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:02 smithi089 bash[21793]: audit 2024-07-15T06:38:01.375408+0000 mon.a (mon.0) 66 : audit [INF] from='client.? 172.21.15.89:0/1203140001' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-07-15T06:38:02.623 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:02 smithi089 bash[21793]: cluster 2024-07-15T06:38:01.375464+0000 mon.a (mon.0) 67 : cluster [DBG] mgrmap e9: y(active, since 25s) 2024-07-15T06:38:03.123 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:38:02 smithi089 bash[22045]: debug 2024-07-15T06:38:02.741+0000 7ff83263bec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-07-15T06:38:03.124 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:38:02 smithi089 bash[22045]: debug 2024-07-15T06:38:02.805+0000 7ff83263bec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-07-15T06:38:03.124 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:38:02 smithi089 bash[22045]: debug 2024-07-15T06:38:02.969+0000 7ff83263bec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-07-15T06:38:03.700 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:03 smithi089 bash[21793]: audit 2024-07-15T06:38:03.389013+0000 mon.a (mon.0) 68 : audit [DBG] from='client.? 172.21.15.89:0/1045117326' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-07-15T06:38:03.701 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:38:03 smithi089 bash[22045]: debug 2024-07-15T06:38:03.629+0000 7ff83263bec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-07-15T06:38:03.944 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout { 2024-07-15T06:38:03.944 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "epoch": 9, 2024-07-15T06:38:03.944 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "available": true, 2024-07-15T06:38:03.945 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "active_name": "y", 2024-07-15T06:38:03.945 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "num_standby": 0 2024-07-15T06:38:03.945 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout } 2024-07-15T06:38:03.945 INFO:teuthology.orchestra.run.smithi089.stdout:Waiting for the mgr to restart... 2024-07-15T06:38:03.945 INFO:teuthology.orchestra.run.smithi089.stdout:Waiting for mgr epoch 9... 2024-07-15T06:38:04.014 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:38:03 smithi089 bash[22045]: debug 2024-07-15T06:38:03.697+0000 7ff83263bec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-07-15T06:38:04.014 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:38:03 smithi089 bash[22045]: debug 2024-07-15T06:38:03.765+0000 7ff83263bec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-07-15T06:38:04.014 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:38:03 smithi089 bash[22045]: debug 2024-07-15T06:38:03.925+0000 7ff83263bec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-07-15T06:38:04.314 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:38:04 smithi089 bash[22045]: debug 2024-07-15T06:38:04.013+0000 7ff83263bec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-07-15T06:38:04.314 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:38:04 smithi089 bash[22045]: debug 2024-07-15T06:38:04.081+0000 7ff83263bec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-07-15T06:38:04.314 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:38:04 smithi089 bash[22045]: debug 2024-07-15T06:38:04.309+0000 7ff83263bec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-07-15T06:38:04.623 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:38:04 smithi089 bash[22045]: debug 2024-07-15T06:38:04.377+0000 7ff83263bec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-07-15T06:38:04.624 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:38:04 smithi089 bash[22045]: debug 2024-07-15T06:38:04.553+0000 7ff83263bec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-07-15T06:38:05.032 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:38:04 smithi089 bash[22045]: debug 2024-07-15T06:38:04.629+0000 7ff83263bec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-07-15T06:38:05.032 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:38:04 smithi089 bash[22045]: debug 2024-07-15T06:38:04.865+0000 7ff83263bec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-07-15T06:38:05.373 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:38:05 smithi089 bash[22045]: debug 2024-07-15T06:38:05.029+0000 7ff83263bec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-07-15T06:38:05.374 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:38:05 smithi089 bash[22045]: debug 2024-07-15T06:38:05.109+0000 7ff83263bec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-07-15T06:38:05.374 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:38:05 smithi089 bash[22045]: debug 2024-07-15T06:38:05.265+0000 7ff83263bec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-07-15T06:38:06.123 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:38:06 smithi089 bash[22045]: debug 2024-07-15T06:38:06.065+0000 7ff83263bec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-07-15T06:38:06.423 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:38:06 smithi089 bash[22045]: debug 2024-07-15T06:38:06.141+0000 7ff83263bec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-07-15T06:38:06.423 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:38:06 smithi089 bash[22045]: debug 2024-07-15T06:38:06.345+0000 7ff83263bec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-07-15T06:38:06.423 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:38:06 smithi089 bash[22045]: debug 2024-07-15T06:38:06.421+0000 7ff83263bec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-07-15T06:38:07.031 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:38:06 smithi089 bash[22045]: debug 2024-07-15T06:38:06.737+0000 7ff83263bec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-07-15T06:38:07.031 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:38:06 smithi089 bash[22045]: debug 2024-07-15T06:38:06.817+0000 7ff83263bec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-07-15T06:38:07.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:06 smithi089 bash[21793]: cluster 2024-07-15T06:38:06.841108+0000 mon.a (mon.0) 69 : cluster [INF] Active manager daemon y restarted 2024-07-15T06:38:07.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:06 smithi089 bash[21793]: cluster 2024-07-15T06:38:06.841519+0000 mon.a (mon.0) 70 : cluster [INF] Activating manager daemon y 2024-07-15T06:38:07.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:06 smithi089 bash[21793]: cluster 2024-07-15T06:38:06.850758+0000 mon.a (mon.0) 71 : cluster [DBG] osdmap e3: 0 total, 0 up, 0 in 2024-07-15T06:38:07.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:06 smithi089 bash[21793]: cluster 2024-07-15T06:38:06.850956+0000 mon.a (mon.0) 72 : cluster [DBG] mgrmap e10: y(active, starting, since 0.00960225s) 2024-07-15T06:38:07.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:06 smithi089 bash[21793]: audit 2024-07-15T06:38:06.851813+0000 mon.a (mon.0) 73 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-07-15T06:38:07.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:06 smithi089 bash[21793]: audit 2024-07-15T06:38:06.853458+0000 mon.a (mon.0) 74 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-07-15T06:38:07.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:06 smithi089 bash[21793]: audit 2024-07-15T06:38:06.854171+0000 mon.a (mon.0) 75 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-07-15T06:38:07.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:06 smithi089 bash[21793]: audit 2024-07-15T06:38:06.854284+0000 mon.a (mon.0) 76 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-07-15T06:38:07.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:06 smithi089 bash[21793]: audit 2024-07-15T06:38:06.854385+0000 mon.a (mon.0) 77 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-07-15T06:38:07.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:06 smithi089 bash[21793]: cluster 2024-07-15T06:38:06.877358+0000 mon.a (mon.0) 78 : cluster [INF] Manager daemon y is now available 2024-07-15T06:38:07.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:06 smithi089 bash[21793]: audit 2024-07-15T06:38:06.891906+0000 mon.a (mon.0) 79 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:38:08.123 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:07 smithi089 bash[21793]: audit 2024-07-15T06:38:06.895920+0000 mon.a (mon.0) 80 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:38:08.124 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:07 smithi089 bash[21793]: audit 2024-07-15T06:38:06.911202+0000 mon.a (mon.0) 81 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-07-15T06:38:08.124 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:07 smithi089 bash[21793]: audit 2024-07-15T06:38:06.913805+0000 mon.a (mon.0) 82 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-07-15T06:38:08.124 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:07 smithi089 bash[21793]: cluster 2024-07-15T06:38:07.853059+0000 mon.a (mon.0) 83 : cluster [DBG] mgrmap e11: y(active, since 1.01171s) 2024-07-15T06:38:08.460 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout { 2024-07-15T06:38:08.460 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 11, 2024-07-15T06:38:08.460 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "initialized": true 2024-07-15T06:38:08.461 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout } 2024-07-15T06:38:08.461 INFO:teuthology.orchestra.run.smithi089.stdout:mgr epoch 9 is available 2024-07-15T06:38:08.461 INFO:teuthology.orchestra.run.smithi089.stdout:Generating a dashboard self-signed certificate... 2024-07-15T06:38:08.553 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:38:08 smithi089 bash[22045]: debug 2024-07-15T06:38:08.189+0000 7ff7e3262640 -1 log_channel(cephadm) log [ERR] : [15/Jul/2024:06:38:08] ENGINE Error in HTTPServer.serve 2024-07-15T06:38:08.553 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:38:08 smithi089 bash[22045]: Traceback (most recent call last): 2024-07-15T06:38:08.553 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:38:08 smithi089 bash[22045]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-07-15T06:38:08.553 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:38:08 smithi089 bash[22045]: self._connections.run(self.expiration_interval) 2024-07-15T06:38:08.554 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:38:08 smithi089 bash[22045]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-07-15T06:38:08.554 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:38:08 smithi089 bash[22045]: self._run(expiration_interval) 2024-07-15T06:38:08.554 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:38:08 smithi089 bash[22045]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-07-15T06:38:08.554 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:38:08 smithi089 bash[22045]: new_conn = self._from_server_socket(self.server.socket) 2024-07-15T06:38:08.554 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:38:08 smithi089 bash[22045]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-07-15T06:38:08.554 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:38:08 smithi089 bash[22045]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-07-15T06:38:08.554 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:38:08 smithi089 bash[22045]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-07-15T06:38:08.554 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:38:08 smithi089 bash[22045]: s = self.context.wrap_socket( 2024-07-15T06:38:08.554 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:38:08 smithi089 bash[22045]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-07-15T06:38:08.554 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:38:08 smithi089 bash[22045]: return self.sslsocket_class._create( 2024-07-15T06:38:08.554 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:38:08 smithi089 bash[22045]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-07-15T06:38:08.554 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:38:08 smithi089 bash[22045]: self.do_handshake() 2024-07-15T06:38:08.555 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:38:08 smithi089 bash[22045]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-07-15T06:38:08.555 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:38:08 smithi089 bash[22045]: self._sslobj.do_handshake() 2024-07-15T06:38:08.555 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:38:08 smithi089 bash[22045]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-07-15T06:38:09.456 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:09 smithi089 bash[21793]: cephadm 2024-07-15T06:38:08.069825+0000 mgr.y (mgr.14150) 3 : cephadm [INF] [15/Jul/2024:06:38:08] ENGINE Bus STARTING 2024-07-15T06:38:09.456 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:09 smithi089 bash[21793]: cephadm 2024-07-15T06:38:08.189236+0000 mgr.y (mgr.14150) 4 : cephadm [INF] [15/Jul/2024:06:38:08] ENGINE Serving on https://172.21.15.89:7150 2024-07-15T06:38:09.456 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:09 smithi089 bash[21793]: cephadm 2024-07-15T06:38:08.189333+0000 mgr.y (mgr.14150) 5 : cephadm [INF] [15/Jul/2024:06:38:08] ENGINE Bus STARTED 2024-07-15T06:38:09.456 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:09 smithi089 bash[21793]: cephadm 2024-07-15T06:38:08.191543+0000 mgr.y (mgr.14150) 6 : cephadm [ERR] [15/Jul/2024:06:38:08] ENGINE Error in HTTPServer.serve 2024-07-15T06:38:09.456 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:09 smithi089 bash[21793]: Traceback (most recent call last): 2024-07-15T06:38:09.456 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:09 smithi089 bash[21793]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-07-15T06:38:09.456 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:09 smithi089 bash[21793]: self._connections.run(self.expiration_interval) 2024-07-15T06:38:09.456 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:09 smithi089 bash[21793]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-07-15T06:38:09.456 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:09 smithi089 bash[21793]: self._run(expiration_interval) 2024-07-15T06:38:09.456 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:09 smithi089 bash[21793]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-07-15T06:38:09.456 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:09 smithi089 bash[21793]: new_conn = self._from_server_socket(self.server.socket) 2024-07-15T06:38:09.457 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:09 smithi089 bash[21793]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-07-15T06:38:09.457 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:09 smithi089 bash[21793]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-07-15T06:38:09.457 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:09 smithi089 bash[21793]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-07-15T06:38:09.457 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:09 smithi089 bash[21793]: s = self.context.wrap_socket( 2024-07-15T06:38:09.457 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:09 smithi089 bash[21793]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-07-15T06:38:09.457 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:09 smithi089 bash[21793]: return self.sslsocket_class._create( 2024-07-15T06:38:09.457 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:09 smithi089 bash[21793]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-07-15T06:38:09.458 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:09 smithi089 bash[21793]: self.do_handshake() 2024-07-15T06:38:09.458 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:09 smithi089 bash[21793]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-07-15T06:38:09.458 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:09 smithi089 bash[21793]: self._sslobj.do_handshake() 2024-07-15T06:38:09.458 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:09 smithi089 bash[21793]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-07-15T06:38:09.458 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:09 smithi089 bash[21793]: audit 2024-07-15T06:38:08.196386+0000 mon.a (mon.0) 84 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:38:10.373 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:10 smithi089 bash[21793]: cluster 2024-07-15T06:38:09.203134+0000 mon.a (mon.0) 85 : cluster [DBG] mgrmap e12: y(active, since 2s) 2024-07-15T06:38:10.374 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:10 smithi089 bash[21793]: audit 2024-07-15T06:38:10.005786+0000 mon.a (mon.0) 86 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:38:10.374 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:10 smithi089 bash[21793]: audit 2024-07-15T06:38:10.010905+0000 mon.a (mon.0) 87 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:38:10.518 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout Self-signed certificate created 2024-07-15T06:38:10.519 INFO:teuthology.orchestra.run.smithi089.stdout:Creating initial admin user... 2024-07-15T06:38:11.623 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:11 smithi089 bash[21793]: audit 2024-07-15T06:38:09.948622+0000 mgr.y (mgr.14150) 7 : audit [DBG] from='client.14162 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T06:38:12.780 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout {"username": "admin", "password": "$2b$12$kUFDvfT1VFWPyMqA33VJReEZFYn4c1kC2kIqpxbM7zHavsvR72gD2", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1721025492, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-07-15T06:38:12.780 INFO:teuthology.orchestra.run.smithi089.stdout:Fetching dashboard port number... 2024-07-15T06:38:13.624 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:13 smithi089 bash[21793]: audit 2024-07-15T06:38:11.922471+0000 mgr.y (mgr.14150) 8 : audit [DBG] from='client.14164 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T06:38:13.624 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:13 smithi089 bash[21793]: audit 2024-07-15T06:38:12.163550+0000 mon.a (mon.0) 88 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:38:13.624 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:13 smithi089 bash[21793]: audit 2024-07-15T06:38:12.228380+0000 mon.a (mon.0) 89 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:38:13.624 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:13 smithi089 bash[21793]: audit 2024-07-15T06:38:12.764179+0000 mon.a (mon.0) 90 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:38:14.775 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout 8443 2024-07-15T06:38:14.775 INFO:teuthology.orchestra.run.smithi089.stdout:firewalld does not appear to be present 2024-07-15T06:38:14.775 INFO:teuthology.orchestra.run.smithi089.stdout:Not possible to open ports <[8443]>. firewalld.service is not available 2024-07-15T06:38:14.777 INFO:teuthology.orchestra.run.smithi089.stdout:Ceph Dashboard is now available at: 2024-07-15T06:38:14.777 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-07-15T06:38:14.777 INFO:teuthology.orchestra.run.smithi089.stdout: URL: https://smithi089.front.sepia.ceph.com:8443/ 2024-07-15T06:38:14.777 INFO:teuthology.orchestra.run.smithi089.stdout: User: admin 2024-07-15T06:38:14.777 INFO:teuthology.orchestra.run.smithi089.stdout: Password: ewjajst6l4 2024-07-15T06:38:14.777 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-07-15T06:38:14.778 INFO:teuthology.orchestra.run.smithi089.stdout:Saving cluster configuration to /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/config directory 2024-07-15T06:38:14.778 INFO:teuthology.orchestra.run.smithi089.stdout:Enabling autotune for osd_memory_target 2024-07-15T06:38:15.124 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:14 smithi089 bash[21793]: cluster 2024-07-15T06:38:13.767160+0000 mon.a (mon.0) 91 : cluster [DBG] mgrmap e13: y(active, since 6s) 2024-07-15T06:38:15.124 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:14 smithi089 bash[21793]: audit 2024-07-15T06:38:14.225601+0000 mon.a (mon.0) 92 : audit [DBG] from='client.? 172.21.15.89:0/72421938' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-07-15T06:38:18.878 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stderr set mgr/dashboard/cluster/status 2024-07-15T06:38:18.879 INFO:teuthology.orchestra.run.smithi089.stdout:You can access the Ceph CLI as following in case of multi-cluster or non-default config: 2024-07-15T06:38:18.879 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-07-15T06:38:18.879 INFO:teuthology.orchestra.run.smithi089.stdout: sudo /home/ubuntu/cephtest/cephadm shell --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-07-15T06:38:18.879 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-07-15T06:38:18.879 INFO:teuthology.orchestra.run.smithi089.stdout:Or, if you are only running a single cluster on this host: 2024-07-15T06:38:18.879 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-07-15T06:38:18.879 INFO:teuthology.orchestra.run.smithi089.stdout: sudo /home/ubuntu/cephtest/cephadm shell 2024-07-15T06:38:18.879 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-07-15T06:38:18.879 INFO:teuthology.orchestra.run.smithi089.stdout:Please consider enabling telemetry to help improve Ceph: 2024-07-15T06:38:18.879 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-07-15T06:38:18.879 INFO:teuthology.orchestra.run.smithi089.stdout: ceph telemetry on 2024-07-15T06:38:18.880 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-07-15T06:38:18.880 INFO:teuthology.orchestra.run.smithi089.stdout:For more information see: 2024-07-15T06:38:18.880 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-07-15T06:38:18.880 INFO:teuthology.orchestra.run.smithi089.stdout: https://docs.ceph.com/docs/master/mgr/telemetry/ 2024-07-15T06:38:18.880 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-07-15T06:38:18.880 INFO:teuthology.orchestra.run.smithi089.stdout:Bootstrap complete. 2024-07-15T06:38:18.918 INFO:tasks.cephadm:Fetching config... 2024-07-15T06:38:18.918 DEBUG:teuthology.orchestra.run.smithi089:> set -ex 2024-07-15T06:38:18.918 DEBUG:teuthology.orchestra.run.smithi089:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-07-15T06:38:18.923 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-07-15T06:38:18.923 DEBUG:teuthology.orchestra.run.smithi089:> set -ex 2024-07-15T06:38:18.923 DEBUG:teuthology.orchestra.run.smithi089:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-07-15T06:38:18.969 INFO:tasks.cephadm:Fetching mon keyring... 2024-07-15T06:38:18.969 DEBUG:teuthology.orchestra.run.smithi089:> set -ex 2024-07-15T06:38:18.969 DEBUG:teuthology.orchestra.run.smithi089:> sudo dd if=/var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/keyring of=/dev/stdout 2024-07-15T06:38:19.023 INFO:tasks.cephadm:Fetching pub ssh key... 2024-07-15T06:38:19.023 DEBUG:teuthology.orchestra.run.smithi089:> set -ex 2024-07-15T06:38:19.023 DEBUG:teuthology.orchestra.run.smithi089:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-07-15T06:38:19.069 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-07-15T06:38:19.069 DEBUG:teuthology.orchestra.run.smithi089:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCT4NEbULOb3GWmHnmWxk83Z/qKS+P3BoKW8JNgOvff/ZN904M4hlCC+5mHyx8fpMoaJ8U/+yhaCGOH+yHADlO+1f371CUVoJJbGJ0TF8QR95Qkn0fhWZFhDJoNP6thKcqshDZty4Mlacbccu5ZRGFDv/7yxTNlweDDkUVUil4gdLowCLdiQujGxXzY4skNVhoG+d+ZeMSttrKxgoSOfsZWSAB0+KeobWOTNlZYWmrpfABQneb/PxqKk6QRz0gKFYx7N7R8ToAAKbVZ544n86yST8Wi6NUN9FU9gmk4qAFxOLH5SoM8dpwukVjp8MxL9tPqIBjevpwUT1mdWAi5WczCmti3ncM4xRNutBzfk9T4AuDL/WHkG+n+JeTpuJX1tl9S+ByqVi9IjCEx0qBFbGuCRagKSoTbslyWxNdOFXYZwHLvEgfENiFL6c5omX7q6Itv+p9M/Il8tjCojwEmiIMxzOpJ9DtbufYRO/jHYcMdcrOnMAue//dHypGc+qwRjs8= ceph-75ba02bc-4274-11ef-bcad-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-07-15T06:38:19.127 INFO:teuthology.orchestra.run.smithi089.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCT4NEbULOb3GWmHnmWxk83Z/qKS+P3BoKW8JNgOvff/ZN904M4hlCC+5mHyx8fpMoaJ8U/+yhaCGOH+yHADlO+1f371CUVoJJbGJ0TF8QR95Qkn0fhWZFhDJoNP6thKcqshDZty4Mlacbccu5ZRGFDv/7yxTNlweDDkUVUil4gdLowCLdiQujGxXzY4skNVhoG+d+ZeMSttrKxgoSOfsZWSAB0+KeobWOTNlZYWmrpfABQneb/PxqKk6QRz0gKFYx7N7R8ToAAKbVZ544n86yST8Wi6NUN9FU9gmk4qAFxOLH5SoM8dpwukVjp8MxL9tPqIBjevpwUT1mdWAi5WczCmti3ncM4xRNutBzfk9T4AuDL/WHkG+n+JeTpuJX1tl9S+ByqVi9IjCEx0qBFbGuCRagKSoTbslyWxNdOFXYZwHLvEgfENiFL6c5omX7q6Itv+p9M/Il8tjCojwEmiIMxzOpJ9DtbufYRO/jHYcMdcrOnMAue//dHypGc+qwRjs8= ceph-75ba02bc-4274-11ef-bcad-c7b262605968 2024-07-15T06:38:19.134 DEBUG:teuthology.orchestra.run.smithi134:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCT4NEbULOb3GWmHnmWxk83Z/qKS+P3BoKW8JNgOvff/ZN904M4hlCC+5mHyx8fpMoaJ8U/+yhaCGOH+yHADlO+1f371CUVoJJbGJ0TF8QR95Qkn0fhWZFhDJoNP6thKcqshDZty4Mlacbccu5ZRGFDv/7yxTNlweDDkUVUil4gdLowCLdiQujGxXzY4skNVhoG+d+ZeMSttrKxgoSOfsZWSAB0+KeobWOTNlZYWmrpfABQneb/PxqKk6QRz0gKFYx7N7R8ToAAKbVZ544n86yST8Wi6NUN9FU9gmk4qAFxOLH5SoM8dpwukVjp8MxL9tPqIBjevpwUT1mdWAi5WczCmti3ncM4xRNutBzfk9T4AuDL/WHkG+n+JeTpuJX1tl9S+ByqVi9IjCEx0qBFbGuCRagKSoTbslyWxNdOFXYZwHLvEgfENiFL6c5omX7q6Itv+p9M/Il8tjCojwEmiIMxzOpJ9DtbufYRO/jHYcMdcrOnMAue//dHypGc+qwRjs8= ceph-75ba02bc-4274-11ef-bcad-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-07-15T06:38:19.156 INFO:teuthology.orchestra.run.smithi134.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCT4NEbULOb3GWmHnmWxk83Z/qKS+P3BoKW8JNgOvff/ZN904M4hlCC+5mHyx8fpMoaJ8U/+yhaCGOH+yHADlO+1f371CUVoJJbGJ0TF8QR95Qkn0fhWZFhDJoNP6thKcqshDZty4Mlacbccu5ZRGFDv/7yxTNlweDDkUVUil4gdLowCLdiQujGxXzY4skNVhoG+d+ZeMSttrKxgoSOfsZWSAB0+KeobWOTNlZYWmrpfABQneb/PxqKk6QRz0gKFYx7N7R8ToAAKbVZ544n86yST8Wi6NUN9FU9gmk4qAFxOLH5SoM8dpwukVjp8MxL9tPqIBjevpwUT1mdWAi5WczCmti3ncM4xRNutBzfk9T4AuDL/WHkG+n+JeTpuJX1tl9S+ByqVi9IjCEx0qBFbGuCRagKSoTbslyWxNdOFXYZwHLvEgfENiFL6c5omX7q6Itv+p9M/Il8tjCojwEmiIMxzOpJ9DtbufYRO/jHYcMdcrOnMAue//dHypGc+qwRjs8= ceph-75ba02bc-4274-11ef-bcad-c7b262605968 2024-07-15T06:38:19.166 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-07-15T06:38:19.188 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:19 smithi089 bash[21793]: audit 2024-07-15T06:38:18.077843+0000 mon.a (mon.0) 93 : audit [INF] from='client.? 172.21.15.89:0/3103848781' entity='client.admin' 2024-07-15T06:38:25.093 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T06:38:25.952 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:25 smithi089 bash[21793]: audit 2024-07-15T06:38:24.694567+0000 mon.a (mon.0) 94 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:38:25.952 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:25 smithi089 bash[21793]: audit 2024-07-15T06:38:24.699375+0000 mon.a (mon.0) 95 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:38:25.952 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:25 smithi089 bash[21793]: audit 2024-07-15T06:38:24.700753+0000 mon.a (mon.0) 96 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi089", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:38:25.952 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:25 smithi089 bash[21793]: audit 2024-07-15T06:38:24.707710+0000 mon.a (mon.0) 97 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:38:25.952 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:25 smithi089 bash[21793]: audit 2024-07-15T06:38:24.735159+0000 mon.a (mon.0) 98 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:38:25.952 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:25 smithi089 bash[21793]: audit 2024-07-15T06:38:24.742577+0000 mon.a (mon.0) 99 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:38:28.184 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:27 smithi089 bash[21793]: cluster 2024-07-15T06:38:26.855895+0000 mgr.y (mgr.14150) 9 : cluster [DBG] pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:38:28.599 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-07-15T06:38:28.599 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-07-15T06:38:29.374 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:29 smithi089 bash[21793]: audit 2024-07-15T06:38:28.003872+0000 mon.a (mon.0) 100 : audit [INF] from='client.? 172.21.15.89:0/2500243150' entity='client.admin' 2024-07-15T06:38:29.374 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:29 smithi089 bash[21793]: audit 2024-07-15T06:38:28.016212+0000 mon.a (mon.0) 101 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:38:29.374 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:29 smithi089 bash[21793]: audit 2024-07-15T06:38:28.024300+0000 mon.a (mon.0) 102 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:38:30.373 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:30 smithi089 bash[21793]: cluster 2024-07-15T06:38:28.856256+0000 mgr.y (mgr.14150) 10 : cluster [DBG] pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:38:32.374 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:32 smithi089 bash[21793]: cluster 2024-07-15T06:38:30.856632+0000 mgr.y (mgr.14150) 11 : cluster [DBG] pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:38:33.346 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T06:38:34.374 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:34 smithi089 bash[21793]: cluster 2024-07-15T06:38:32.857017+0000 mgr.y (mgr.14150) 12 : cluster [DBG] pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:38:36.317 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:36 smithi089 bash[21793]: cluster 2024-07-15T06:38:34.857454+0000 mgr.y (mgr.14150) 13 : cluster [DBG] pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:38:36.915 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi134 2024-07-15T06:38:36.915 DEBUG:teuthology.orchestra.run.smithi134:> set -ex 2024-07-15T06:38:36.915 DEBUG:teuthology.orchestra.run.smithi134:> dd of=/etc/ceph/ceph.conf 2024-07-15T06:38:36.921 DEBUG:teuthology.orchestra.run.smithi134:> set -ex 2024-07-15T06:38:36.921 DEBUG:teuthology.orchestra.run.smithi134:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-07-15T06:38:36.969 INFO:tasks.cephadm:Adding host smithi134 to orchestrator... 2024-07-15T06:38:36.969 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph orch host add smithi134 2024-07-15T06:38:37.624 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:37 smithi089 bash[21793]: audit 2024-07-15T06:38:36.297948+0000 mgr.y (mgr.14150) 14 : audit [DBG] from='client.14174 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T06:38:37.624 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:37 smithi089 bash[21793]: audit 2024-07-15T06:38:36.305161+0000 mon.a (mon.0) 103 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:38:37.624 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:37 smithi089 bash[21793]: audit 2024-07-15T06:38:36.306260+0000 mon.a (mon.0) 104 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:38:37.625 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:37 smithi089 bash[21793]: audit 2024-07-15T06:38:36.308263+0000 mon.a (mon.0) 105 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:38:37.625 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:37 smithi089 bash[21793]: audit 2024-07-15T06:38:36.309270+0000 mon.a (mon.0) 106 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:38:37.625 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:37 smithi089 bash[21793]: cephadm 2024-07-15T06:38:36.311047+0000 mgr.y (mgr.14150) 15 : cephadm [INF] Updating smithi089:/etc/ceph/ceph.conf 2024-07-15T06:38:37.625 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:37 smithi089 bash[21793]: cephadm 2024-07-15T06:38:36.403623+0000 mgr.y (mgr.14150) 16 : cephadm [INF] Updating smithi089:/var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/config/ceph.conf 2024-07-15T06:38:37.625 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:37 smithi089 bash[21793]: audit 2024-07-15T06:38:36.641001+0000 mon.a (mon.0) 107 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:38:37.625 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:37 smithi089 bash[21793]: audit 2024-07-15T06:38:36.645993+0000 mon.a (mon.0) 108 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:38:37.625 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:37 smithi089 bash[21793]: audit 2024-07-15T06:38:36.648800+0000 mon.a (mon.0) 109 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:38:38.624 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:38 smithi089 bash[21793]: cephadm 2024-07-15T06:38:36.479629+0000 mgr.y (mgr.14150) 17 : cephadm [INF] Updating smithi089:/etc/ceph/ceph.client.admin.keyring 2024-07-15T06:38:38.624 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:38 smithi089 bash[21793]: cephadm 2024-07-15T06:38:36.560182+0000 mgr.y (mgr.14150) 18 : cephadm [INF] Updating smithi089:/var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/config/ceph.client.admin.keyring 2024-07-15T06:38:38.624 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:38 smithi089 bash[21793]: cluster 2024-07-15T06:38:36.857875+0000 mgr.y (mgr.14150) 19 : cluster [DBG] pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:38:40.624 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:40 smithi089 bash[21793]: cluster 2024-07-15T06:38:38.858285+0000 mgr.y (mgr.14150) 20 : cluster [DBG] pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:38:41.678 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T06:38:42.624 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:42 smithi089 bash[21793]: cluster 2024-07-15T06:38:40.858680+0000 mgr.y (mgr.14150) 21 : cluster [DBG] pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:38:44.624 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:44 smithi089 bash[21793]: cluster 2024-07-15T06:38:42.859142+0000 mgr.y (mgr.14150) 22 : cluster [DBG] pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:38:46.624 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:46 smithi089 bash[21793]: audit 2024-07-15T06:38:44.582460+0000 mgr.y (mgr.14150) 23 : audit [DBG] from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi134", "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T06:38:46.624 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:46 smithi089 bash[21793]: cluster 2024-07-15T06:38:44.859531+0000 mgr.y (mgr.14150) 24 : cluster [DBG] pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:38:47.624 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:47 smithi089 bash[21793]: cephadm 2024-07-15T06:38:45.631430+0000 mgr.y (mgr.14150) 25 : cephadm [INF] Deploying cephadm binary to smithi134 2024-07-15T06:38:47.870 INFO:teuthology.orchestra.run.smithi089.stdout:Added host 'smithi134' with addr '172.21.15.134' 2024-07-15T06:38:48.444 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph orch host ls --format=json 2024-07-15T06:38:48.624 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:48 smithi089 bash[21793]: cluster 2024-07-15T06:38:46.859819+0000 mgr.y (mgr.14150) 26 : cluster [DBG] pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:38:48.624 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:48 smithi089 bash[21793]: audit 2024-07-15T06:38:47.869432+0000 mon.a (mon.0) 110 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:38:48.625 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:48 smithi089 bash[21793]: audit 2024-07-15T06:38:47.870968+0000 mon.a (mon.0) 111 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:38:48.625 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:48 smithi089 bash[21793]: audit 2024-07-15T06:38:48.113717+0000 mon.a (mon.0) 112 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:38:49.624 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:49 smithi089 bash[21793]: cephadm 2024-07-15T06:38:47.870230+0000 mgr.y (mgr.14150) 27 : cephadm [INF] Added host smithi134 2024-07-15T06:38:50.874 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:50 smithi089 bash[21793]: cluster 2024-07-15T06:38:48.860151+0000 mgr.y (mgr.14150) 28 : cluster [DBG] pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:38:50.874 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:50 smithi089 bash[21793]: audit 2024-07-15T06:38:49.378552+0000 mon.a (mon.0) 113 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:38:50.875 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:50 smithi089 bash[21793]: audit 2024-07-15T06:38:50.067812+0000 mon.a (mon.0) 114 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:38:52.874 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:52 smithi089 bash[21793]: cluster 2024-07-15T06:38:50.860586+0000 mgr.y (mgr.14150) 29 : cluster [DBG] pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:38:53.155 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T06:38:54.874 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:54 smithi089 bash[21793]: cluster 2024-07-15T06:38:52.860997+0000 mgr.y (mgr.14150) 30 : cluster [DBG] pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:38:56.195 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-07-15T06:38:56.195 INFO:teuthology.orchestra.run.smithi089.stdout:[{"addr": "172.21.15.89", "hostname": "smithi089", "labels": [], "status": ""}, {"addr": "172.21.15.134", "hostname": "smithi134", "labels": [], "status": ""}] 2024-07-15T06:38:56.625 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:56 smithi089 bash[21793]: cluster 2024-07-15T06:38:54.861476+0000 mgr.y (mgr.14150) 31 : cluster [DBG] pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:38:56.625 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:56 smithi089 bash[21793]: audit 2024-07-15T06:38:55.631996+0000 mon.a (mon.0) 115 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:38:56.625 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:56 smithi089 bash[21793]: audit 2024-07-15T06:38:55.639547+0000 mon.a (mon.0) 116 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:38:56.625 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:56 smithi089 bash[21793]: audit 2024-07-15T06:38:55.648078+0000 mon.a (mon.0) 117 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:38:56.625 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:56 smithi089 bash[21793]: audit 2024-07-15T06:38:55.655792+0000 mon.a (mon.0) 118 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:38:56.625 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:56 smithi089 bash[21793]: audit 2024-07-15T06:38:55.657223+0000 mon.a (mon.0) 119 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi134", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:38:56.625 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:56 smithi089 bash[21793]: audit 2024-07-15T06:38:55.659171+0000 mon.a (mon.0) 120 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:38:56.625 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:56 smithi089 bash[21793]: audit 2024-07-15T06:38:55.660786+0000 mon.a (mon.0) 121 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:38:56.625 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:56 smithi089 bash[21793]: audit 2024-07-15T06:38:56.171394+0000 mon.a (mon.0) 122 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:38:56.625 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:56 smithi089 bash[21793]: audit 2024-07-15T06:38:56.178840+0000 mon.a (mon.0) 123 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:38:56.626 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:56 smithi089 bash[21793]: audit 2024-07-15T06:38:56.187137+0000 mon.a (mon.0) 124 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:38:56.755 INFO:tasks.cephadm:Setting crush tunables to default 2024-07-15T06:38:56.755 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph osd crush tunables default 2024-07-15T06:38:57.874 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:57 smithi089 bash[21793]: cephadm 2024-07-15T06:38:55.663031+0000 mgr.y (mgr.14150) 32 : cephadm [INF] Updating smithi134:/etc/ceph/ceph.conf 2024-07-15T06:38:57.875 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:57 smithi089 bash[21793]: cephadm 2024-07-15T06:38:55.817434+0000 mgr.y (mgr.14150) 33 : cephadm [INF] Updating smithi134:/var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/config/ceph.conf 2024-07-15T06:38:57.875 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:57 smithi089 bash[21793]: cephadm 2024-07-15T06:38:55.927217+0000 mgr.y (mgr.14150) 34 : cephadm [INF] Updating smithi134:/etc/ceph/ceph.client.admin.keyring 2024-07-15T06:38:57.875 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:57 smithi089 bash[21793]: cephadm 2024-07-15T06:38:56.050473+0000 mgr.y (mgr.14150) 35 : cephadm [INF] Updating smithi134:/var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/config/ceph.client.admin.keyring 2024-07-15T06:38:57.875 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:57 smithi089 bash[21793]: audit 2024-07-15T06:38:56.194570+0000 mgr.y (mgr.14150) 36 : audit [DBG] from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-07-15T06:38:58.874 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:38:58 smithi089 bash[21793]: cluster 2024-07-15T06:38:56.861904+0000 mgr.y (mgr.14150) 37 : cluster [DBG] pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:00.875 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:00 smithi089 bash[21793]: cluster 2024-07-15T06:38:58.862306+0000 mgr.y (mgr.14150) 38 : cluster [DBG] pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:01.469 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T06:39:02.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:02 smithi089 bash[21793]: cluster 2024-07-15T06:39:00.862717+0000 mgr.y (mgr.14150) 39 : cluster [DBG] pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:04.874 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:04 smithi089 bash[21793]: cluster 2024-07-15T06:39:02.863123+0000 mgr.y (mgr.14150) 40 : cluster [DBG] pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:05.408 INFO:teuthology.orchestra.run.smithi089.stderr:adjusted tunables profile to default 2024-07-15T06:39:05.874 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:05 smithi089 bash[21793]: audit 2024-07-15T06:39:04.402983+0000 mon.a (mon.0) 125 : audit [INF] from='client.? 172.21.15.89:0/2297113159' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-07-15T06:39:05.971 INFO:tasks.cephadm:Adding mon.a on smithi089 2024-07-15T06:39:05.971 INFO:tasks.cephadm:Adding mon.c on smithi089 2024-07-15T06:39:05.971 INFO:tasks.cephadm:Adding mon.b on smithi134 2024-07-15T06:39:05.972 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph orch apply mon '3;smithi089:172.21.15.89=a;smithi089:[v2:172.21.15.89:3301,v1:172.21.15.89:6790]=c;smithi134:172.21.15.134=b' 2024-07-15T06:39:06.874 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:06 smithi089 bash[21793]: cluster 2024-07-15T06:39:04.863556+0000 mgr.y (mgr.14150) 41 : cluster [DBG] pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:06.875 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:06 smithi089 bash[21793]: audit 2024-07-15T06:39:05.408403+0000 mon.a (mon.0) 126 : audit [INF] from='client.? 172.21.15.89:0/2297113159' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-07-15T06:39:06.875 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:06 smithi089 bash[21793]: cluster 2024-07-15T06:39:05.408553+0000 mon.a (mon.0) 127 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2024-07-15T06:39:07.186 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/config/ceph.conf 2024-07-15T06:39:08.875 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:08 smithi089 bash[21793]: cluster 2024-07-15T06:39:06.863953+0000 mgr.y (mgr.14150) 42 : cluster [DBG] pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:09.959 INFO:teuthology.orchestra.run.smithi134.stdout:Scheduled mon update... 2024-07-15T06:39:10.741 DEBUG:teuthology.orchestra.run.smithi089:mon.c> sudo journalctl -f -n 0 -u ceph-75ba02bc-4274-11ef-bcad-c7b262605968@mon.c.service 2024-07-15T06:39:10.744 DEBUG:teuthology.orchestra.run.smithi134:mon.b> sudo journalctl -f -n 0 -u ceph-75ba02bc-4274-11ef-bcad-c7b262605968@mon.b.service 2024-07-15T06:39:10.746 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-07-15T06:39:10.747 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph mon dump -f json 2024-07-15T06:39:10.753 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:10 smithi089 bash[21793]: cluster 2024-07-15T06:39:08.864362+0000 mgr.y (mgr.14150) 43 : cluster [DBG] pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:10.753 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:10 smithi089 bash[21793]: audit 2024-07-15T06:39:09.958010+0000 mon.a (mon.0) 128 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:10.753 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:10 smithi089 bash[21793]: audit 2024-07-15T06:39:09.959579+0000 mon.a (mon.0) 129 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:39:10.753 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:10 smithi089 bash[21793]: audit 2024-07-15T06:39:09.963084+0000 mon.a (mon.0) 130 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:39:10.753 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:10 smithi089 bash[21793]: audit 2024-07-15T06:39:09.964857+0000 mon.a (mon.0) 131 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:39:10.753 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:10 smithi089 bash[21793]: audit 2024-07-15T06:39:09.972303+0000 mon.a (mon.0) 132 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:10.753 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:10 smithi089 bash[21793]: audit 2024-07-15T06:39:09.976069+0000 mon.a (mon.0) 133 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-07-15T06:39:10.753 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:10 smithi089 bash[21793]: audit 2024-07-15T06:39:09.977708+0000 mon.a (mon.0) 134 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:39:11.875 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:11 smithi089 bash[21793]: audit 2024-07-15T06:39:09.947711+0000 mgr.y (mgr.14150) 44 : audit [DBG] from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi089:172.21.15.89=a;smithi089:[v2:172.21.15.89:3301,v1:172.21.15.89:6790]=c;smithi134:172.21.15.134=b", "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T06:39:11.875 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:11 smithi089 bash[21793]: cephadm 2024-07-15T06:39:09.953036+0000 mgr.y (mgr.14150) 45 : cephadm [INF] Saving service mon spec with placement smithi089:172.21.15.89=a;smithi089:[v2:172.21.15.89:3301,v1:172.21.15.89:6790]=c;smithi134:172.21.15.134=b;count:3 2024-07-15T06:39:11.875 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:11 smithi089 bash[21793]: cephadm 2024-07-15T06:39:09.979604+0000 mgr.y (mgr.14150) 46 : cephadm [INF] Deploying daemon mon.b on smithi134 2024-07-15T06:39:11.991 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/config/ceph.conf 2024-07-15T06:39:12.874 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:12 smithi089 bash[21793]: cluster 2024-07-15T06:39:10.864877+0000 mgr.y (mgr.14150) 47 : cluster [DBG] pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:14.875 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:14 smithi089 bash[21793]: cluster 2024-07-15T06:39:12.865400+0000 mgr.y (mgr.14150) 48 : cluster [DBG] pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:15.833 INFO:teuthology.orchestra.run.smithi134.stderr:dumped monmap epoch 1 2024-07-15T06:39:15.833 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-07-15T06:39:15.833 INFO:teuthology.orchestra.run.smithi134.stdout:{"epoch":1,"fsid":"75ba02bc-4274-11ef-bcad-c7b262605968","modified":"2024-07-15T06:37:05.016290Z","created":"2024-07-15T06:37:05.016290Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:3300","nonce":0},{"type":"v1","addr":"172.21.15.89:6789","nonce":0}]},"addr":"172.21.15.89:6789/0","public_addr":"172.21.15.89:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-07-15T06:39:16.216 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:15 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:39:16.625 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:16 smithi089 bash[21793]: cluster 2024-07-15T06:39:14.865862+0000 mgr.y (mgr.14150) 49 : cluster [DBG] pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:16.625 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:16 smithi089 bash[21793]: audit 2024-07-15T06:39:15.714323+0000 mon.a (mon.0) 135 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:16.625 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:16 smithi089 bash[21793]: audit 2024-07-15T06:39:15.719717+0000 mon.a (mon.0) 136 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:16.625 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:16 smithi089 bash[21793]: audit 2024-07-15T06:39:15.728023+0000 mon.a (mon.0) 137 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:16.625 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:16 smithi089 bash[21793]: audit 2024-07-15T06:39:15.729261+0000 mon.a (mon.0) 138 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-07-15T06:39:16.625 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:16 smithi089 bash[21793]: audit 2024-07-15T06:39:15.730800+0000 mon.a (mon.0) 139 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:39:16.625 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:16 smithi089 bash[21793]: audit 2024-07-15T06:39:15.832631+0000 mon.a (mon.0) 140 : audit [DBG] from='client.? 172.21.15.134:0/953884829' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-07-15T06:39:17.074 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:17 smithi134 bash[24068]: debug 2024-07-15T06:39:17.051+0000 7fc5f54e1900 1 mon.b@-1(???) e0 preinit fsid 75ba02bc-4274-11ef-bcad-c7b262605968 2024-07-15T06:39:17.075 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:17 smithi134 bash[24068]: debug 2024-07-15T06:39:17.071+0000 7fc5ee2b3640 0 mon.b@-1(synchronizing).mds e1 new map 2024-07-15T06:39:17.075 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:17 smithi134 bash[24068]: debug 2024-07-15T06:39:17.071+0000 7fc5ee2b3640 0 mon.b@-1(synchronizing).mds e1 print_map 2024-07-15T06:39:17.075 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:17 smithi134 bash[24068]: e1 2024-07-15T06:39:17.310 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:17 smithi134 bash[24068]: enable_multiple, ever_enabled_multiple: 1,1 2024-07-15T06:39:17.313 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:17 smithi134 bash[24068]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-07-15T06:39:17.313 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:17 smithi134 bash[24068]: legacy client fscid: -1 2024-07-15T06:39:17.313 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:17 smithi134 bash[24068]: 2024-07-15T06:39:17.314 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:17 smithi134 bash[24068]: No filesystems configured 2024-07-15T06:39:17.314 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:17 smithi134 bash[24068]: debug 2024-07-15T06:39:17.071+0000 7fc5ee2b3640 1 mon.b@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-07-15T06:39:17.314 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:17 smithi134 bash[24068]: debug 2024-07-15T06:39:17.071+0000 7fc5ee2b3640 1 mon.b@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-07-15T06:39:17.314 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:17 smithi134 bash[24068]: debug 2024-07-15T06:39:17.071+0000 7fc5ee2b3640 1 mon.b@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-07-15T06:39:17.314 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:17 smithi134 bash[24068]: debug 2024-07-15T06:39:17.071+0000 7fc5ee2b3640 1 mon.b@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-07-15T06:39:17.314 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:17 smithi134 bash[24068]: debug 2024-07-15T06:39:17.071+0000 7fc5ee2b3640 1 mon.b@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-07-15T06:39:17.314 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:17 smithi134 bash[24068]: debug 2024-07-15T06:39:17.071+0000 7fc5ee2b3640 1 mon.b@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-07-15T06:39:17.314 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:17 smithi134 bash[24068]: debug 2024-07-15T06:39:17.071+0000 7fc5ee2b3640 0 mon.b@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2024-07-15T06:39:17.314 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:17 smithi134 bash[24068]: debug 2024-07-15T06:39:17.071+0000 7fc5ee2b3640 0 mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-07-15T06:39:17.314 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:17 smithi134 bash[24068]: debug 2024-07-15T06:39:17.071+0000 7fc5ee2b3640 0 mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-07-15T06:39:17.314 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:17 smithi134 bash[24068]: debug 2024-07-15T06:39:17.071+0000 7fc5ee2b3640 0 mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-07-15T06:39:17.315 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:17 smithi134 bash[24068]: cephadm 2024-07-15T06:38:36.479629+0000 mgr.y (mgr.14150) 17 : cephadm [INF] Updating smithi089:/etc/ceph/ceph.client.admin.keyring 2024-07-15T06:39:17.315 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:17 smithi134 bash[24068]: cephadm 2024-07-15T06:38:36.560182+0000 mgr.y (mgr.14150) 18 : cephadm [INF] Updating smithi089:/var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/config/ceph.client.admin.keyring 2024-07-15T06:39:17.315 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:17 smithi134 bash[24068]: cluster 2024-07-15T06:38:36.857875+0000 mgr.y (mgr.14150) 19 : cluster [DBG] pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:17.315 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:17 smithi134 bash[24068]: cluster 2024-07-15T06:38:38.858285+0000 mgr.y (mgr.14150) 20 : cluster [DBG] pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:17.315 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:17 smithi134 bash[24068]: cluster 2024-07-15T06:38:40.858680+0000 mgr.y (mgr.14150) 21 : cluster [DBG] pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:17.315 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:17 smithi134 bash[24068]: cluster 2024-07-15T06:38:42.859142+0000 mgr.y (mgr.14150) 22 : cluster [DBG] pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:17.315 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:17 smithi134 bash[24068]: audit 2024-07-15T06:38:44.582460+0000 mgr.y (mgr.14150) 23 : audit [DBG] from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi134", "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T06:39:17.315 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:17 smithi134 bash[24068]: cluster 2024-07-15T06:38:44.859531+0000 mgr.y (mgr.14150) 24 : cluster [DBG] pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:17.315 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:17 smithi134 bash[24068]: cephadm 2024-07-15T06:38:45.631430+0000 mgr.y (mgr.14150) 25 : cephadm [INF] Deploying cephadm binary to smithi134 2024-07-15T06:39:17.316 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:17 smithi134 bash[24068]: cluster 2024-07-15T06:38:46.859819+0000 mgr.y (mgr.14150) 26 : cluster [DBG] pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:17.316 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:17 smithi134 bash[24068]: audit 2024-07-15T06:38:47.869432+0000 mon.a (mon.0) 110 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:17.316 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:17 smithi134 bash[24068]: audit 2024-07-15T06:38:47.870968+0000 mon.a (mon.0) 111 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:39:17.316 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:17 smithi134 bash[24068]: audit 2024-07-15T06:38:48.113717+0000 mon.a (mon.0) 112 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:17.317 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:17 smithi134 bash[24068]: cephadm 2024-07-15T06:38:47.870230+0000 mgr.y (mgr.14150) 27 : cephadm [INF] Added host smithi134 2024-07-15T06:39:17.317 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:17 smithi134 bash[24068]: cluster 2024-07-15T06:38:48.860151+0000 mgr.y (mgr.14150) 28 : cluster [DBG] pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:17.317 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:17 smithi134 bash[24068]: audit 2024-07-15T06:38:49.378552+0000 mon.a (mon.0) 113 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:17.317 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:17 smithi134 bash[24068]: audit 2024-07-15T06:38:50.067812+0000 mon.a (mon.0) 114 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:17.317 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:17 smithi134 bash[24068]: cluster 2024-07-15T06:38:50.860586+0000 mgr.y (mgr.14150) 29 : cluster [DBG] pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:17.317 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:17 smithi134 bash[24068]: cluster 2024-07-15T06:38:52.860997+0000 mgr.y (mgr.14150) 30 : cluster [DBG] pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:17.317 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:17 smithi134 bash[24068]: cluster 2024-07-15T06:38:54.861476+0000 mgr.y (mgr.14150) 31 : cluster [DBG] pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:17.317 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:17 smithi134 bash[24068]: audit 2024-07-15T06:38:55.631996+0000 mon.a (mon.0) 115 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:17.317 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:17 smithi134 bash[24068]: audit 2024-07-15T06:38:55.639547+0000 mon.a (mon.0) 116 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:17.317 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:17 smithi134 bash[24068]: audit 2024-07-15T06:38:55.648078+0000 mon.a (mon.0) 117 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:17.317 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:17 smithi134 bash[24068]: audit 2024-07-15T06:38:55.655792+0000 mon.a (mon.0) 118 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:17.317 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:17 smithi134 bash[24068]: audit 2024-07-15T06:38:55.657223+0000 mon.a (mon.0) 119 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi134", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:39:17.317 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:17 smithi134 bash[24068]: audit 2024-07-15T06:38:55.659171+0000 mon.a (mon.0) 120 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:39:17.318 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:17 smithi134 bash[24068]: audit 2024-07-15T06:38:55.660786+0000 mon.a (mon.0) 121 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:39:17.318 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:17 smithi134 bash[24068]: audit 2024-07-15T06:38:56.171394+0000 mon.a (mon.0) 122 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:17.318 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:17 smithi134 bash[24068]: audit 2024-07-15T06:38:56.178840+0000 mon.a (mon.0) 123 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:17.318 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:17 smithi134 bash[24068]: audit 2024-07-15T06:38:56.187137+0000 mon.a (mon.0) 124 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:17.318 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:17 smithi134 bash[24068]: cephadm 2024-07-15T06:38:55.663031+0000 mgr.y (mgr.14150) 32 : cephadm [INF] Updating smithi134:/etc/ceph/ceph.conf 2024-07-15T06:39:17.318 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:17 smithi134 bash[24068]: cephadm 2024-07-15T06:38:55.817434+0000 mgr.y (mgr.14150) 33 : cephadm [INF] Updating smithi134:/var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/config/ceph.conf 2024-07-15T06:39:17.318 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:17 smithi134 bash[24068]: cephadm 2024-07-15T06:38:55.927217+0000 mgr.y (mgr.14150) 34 : cephadm [INF] Updating smithi134:/etc/ceph/ceph.client.admin.keyring 2024-07-15T06:39:17.318 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:17 smithi134 bash[24068]: cephadm 2024-07-15T06:38:56.050473+0000 mgr.y (mgr.14150) 35 : cephadm [INF] Updating smithi134:/var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/config/ceph.client.admin.keyring 2024-07-15T06:39:17.318 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:17 smithi134 bash[24068]: audit 2024-07-15T06:38:56.194570+0000 mgr.y (mgr.14150) 36 : audit [DBG] from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-07-15T06:39:17.318 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:17 smithi134 bash[24068]: cluster 2024-07-15T06:38:56.861904+0000 mgr.y (mgr.14150) 37 : cluster [DBG] pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:17.318 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:17 smithi134 bash[24068]: cluster 2024-07-15T06:38:58.862306+0000 mgr.y (mgr.14150) 38 : cluster [DBG] pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:17.319 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:17 smithi134 bash[24068]: cluster 2024-07-15T06:39:00.862717+0000 mgr.y (mgr.14150) 39 : cluster [DBG] pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:17.319 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:17 smithi134 bash[24068]: cluster 2024-07-15T06:39:02.863123+0000 mgr.y (mgr.14150) 40 : cluster [DBG] pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:17.319 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:17 smithi134 bash[24068]: audit 2024-07-15T06:39:04.402983+0000 mon.a (mon.0) 125 : audit [INF] from='client.? 172.21.15.89:0/2297113159' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-07-15T06:39:17.319 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:17 smithi134 bash[24068]: cluster 2024-07-15T06:39:04.863556+0000 mgr.y (mgr.14150) 41 : cluster [DBG] pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:17.319 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:17 smithi134 bash[24068]: audit 2024-07-15T06:39:05.408403+0000 mon.a (mon.0) 126 : audit [INF] from='client.? 172.21.15.89:0/2297113159' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-07-15T06:39:17.319 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:17 smithi134 bash[24068]: cluster 2024-07-15T06:39:05.408553+0000 mon.a (mon.0) 127 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2024-07-15T06:39:17.319 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:17 smithi134 bash[24068]: cluster 2024-07-15T06:39:06.863953+0000 mgr.y (mgr.14150) 42 : cluster [DBG] pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:17.319 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:17 smithi134 bash[24068]: cluster 2024-07-15T06:39:08.864362+0000 mgr.y (mgr.14150) 43 : cluster [DBG] pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:17.319 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:17 smithi134 bash[24068]: audit 2024-07-15T06:39:09.958010+0000 mon.a (mon.0) 128 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:17.319 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:17 smithi134 bash[24068]: audit 2024-07-15T06:39:09.959579+0000 mon.a (mon.0) 129 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:39:17.319 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:17 smithi134 bash[24068]: audit 2024-07-15T06:39:09.963084+0000 mon.a (mon.0) 130 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:39:17.319 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:17 smithi134 bash[24068]: audit 2024-07-15T06:39:09.964857+0000 mon.a (mon.0) 131 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:39:17.319 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:17 smithi134 bash[24068]: audit 2024-07-15T06:39:09.972303+0000 mon.a (mon.0) 132 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:17.320 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:17 smithi134 bash[24068]: audit 2024-07-15T06:39:09.976069+0000 mon.a (mon.0) 133 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-07-15T06:39:17.320 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:17 smithi134 bash[24068]: audit 2024-07-15T06:39:09.977708+0000 mon.a (mon.0) 134 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:39:17.320 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:17 smithi134 bash[24068]: audit 2024-07-15T06:39:09.947711+0000 mgr.y (mgr.14150) 44 : audit [DBG] from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi089:172.21.15.89=a;smithi089:[v2:172.21.15.89:3301,v1:172.21.15.89:6790]=c;smithi134:172.21.15.134=b", "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T06:39:17.320 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:17 smithi134 bash[24068]: cephadm 2024-07-15T06:39:09.953036+0000 mgr.y (mgr.14150) 45 : cephadm [INF] Saving service mon spec with placement smithi089:172.21.15.89=a;smithi089:[v2:172.21.15.89:3301,v1:172.21.15.89:6790]=c;smithi134:172.21.15.134=b;count:3 2024-07-15T06:39:17.320 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:17 smithi134 bash[24068]: cephadm 2024-07-15T06:39:09.979604+0000 mgr.y (mgr.14150) 46 : cephadm [INF] Deploying daemon mon.b on smithi134 2024-07-15T06:39:17.320 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:17 smithi134 bash[24068]: cluster 2024-07-15T06:39:10.864877+0000 mgr.y (mgr.14150) 47 : cluster [DBG] pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:17.320 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:17 smithi134 bash[24068]: cluster 2024-07-15T06:39:12.865400+0000 mgr.y (mgr.14150) 48 : cluster [DBG] pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:17.320 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:17 smithi134 bash[24068]: cluster 2024-07-15T06:39:14.865862+0000 mgr.y (mgr.14150) 49 : cluster [DBG] pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:17.320 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:17 smithi134 bash[24068]: audit 2024-07-15T06:39:15.714323+0000 mon.a (mon.0) 135 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:17.320 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:17 smithi134 bash[24068]: audit 2024-07-15T06:39:15.719717+0000 mon.a (mon.0) 136 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:17.320 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:17 smithi134 bash[24068]: audit 2024-07-15T06:39:15.728023+0000 mon.a (mon.0) 137 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:17.320 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:17 smithi134 bash[24068]: audit 2024-07-15T06:39:15.729261+0000 mon.a (mon.0) 138 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-07-15T06:39:17.320 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:17 smithi134 bash[24068]: audit 2024-07-15T06:39:15.730800+0000 mon.a (mon.0) 139 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:39:17.321 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:17 smithi134 bash[24068]: audit 2024-07-15T06:39:15.832631+0000 mon.a (mon.0) 140 : audit [DBG] from='client.? 172.21.15.134:0/953884829' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-07-15T06:39:17.321 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:17 smithi134 bash[24068]: debug 2024-07-15T06:39:17.075+0000 7fc5ee2b3640 1 mon.b@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2024-07-15T06:39:17.820 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-07-15T06:39:17.820 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph mon dump -f json 2024-07-15T06:39:19.478 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:19 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:39:19.478 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:39:19 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:39:19.843 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:19 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:39:19.843 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:39:19 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:39:19.843 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:19 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:39:19.843 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:19 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:39:19.844 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:19 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:39:19.844 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:19 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:39:19.844 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:19 smithi089 systemd[1]: Started Ceph mon.c for 75ba02bc-4274-11ef-bcad-c7b262605968. 2024-07-15T06:39:21.126 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.878+0000 7fd48320d900 0 set uid:gid to 167:167 (ceph:ceph) 2024-07-15T06:39:21.126 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.878+0000 7fd48320d900 0 ceph version 17.2.7-1102-g9a202024 (9a2020246f85463d4fdb115c94aa40b463caa1d2) quincy (stable), process ceph-mon, pid 7 2024-07-15T06:39:21.126 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.878+0000 7fd48320d900 0 pidfile_write: ignore empty --pid-file 2024-07-15T06:39:21.126 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 0 load: jerasure load: lrc 2024-07-15T06:39:21.126 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: RocksDB version: 6.15.5 2024-07-15T06:39:21.126 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-07-15T06:39:21.126 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Compile date Jul 14 2024 2024-07-15T06:39:21.126 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: DB SUMMARY 2024-07-15T06:39:21.126 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: DB Session ID: BBRJRU8AJD2LS0T2WJXB 2024-07-15T06:39:21.126 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: CURRENT file: CURRENT 2024-07-15T06:39:21.127 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: IDENTITY file: IDENTITY 2024-07-15T06:39:21.127 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2024-07-15T06:39:21.127 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 0, files: 2024-07-15T06:39:21.127 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000004.log size: 511 ; 2024-07-15T06:39:21.127 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.error_if_exists: 0 2024-07-15T06:39:21.127 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.create_if_missing: 0 2024-07-15T06:39:21.127 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.paranoid_checks: 1 2024-07-15T06:39:21.127 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-07-15T06:39:21.127 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.env: 0x55d79d097440 2024-07-15T06:39:21.127 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.fs: Posix File System 2024-07-15T06:39:21.127 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.info_log: 0x55d79eb2af40 2024-07-15T06:39:21.127 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.max_file_opening_threads: 16 2024-07-15T06:39:21.127 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.statistics: (nil) 2024-07-15T06:39:21.128 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.use_fsync: 0 2024-07-15T06:39:21.128 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.max_log_file_size: 0 2024-07-15T06:39:21.128 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.max_manifest_file_size: 1073741824 2024-07-15T06:39:21.128 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.log_file_time_to_roll: 0 2024-07-15T06:39:21.128 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.keep_log_file_num: 1000 2024-07-15T06:39:21.128 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.recycle_log_file_num: 0 2024-07-15T06:39:21.129 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.allow_fallocate: 1 2024-07-15T06:39:21.129 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.allow_mmap_reads: 0 2024-07-15T06:39:21.129 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.allow_mmap_writes: 0 2024-07-15T06:39:21.129 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.use_direct_reads: 0 2024-07-15T06:39:21.129 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-07-15T06:39:21.129 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.create_missing_column_families: 0 2024-07-15T06:39:21.129 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.db_log_dir: 2024-07-15T06:39:21.129 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-c/store.db 2024-07-15T06:39:21.129 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.table_cache_numshardbits: 6 2024-07-15T06:39:21.129 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.WAL_ttl_seconds: 0 2024-07-15T06:39:21.130 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.WAL_size_limit_MB: 0 2024-07-15T06:39:21.130 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-07-15T06:39:21.130 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.manifest_preallocation_size: 4194304 2024-07-15T06:39:21.130 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.is_fd_close_on_exec: 1 2024-07-15T06:39:21.130 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.advise_random_on_open: 1 2024-07-15T06:39:21.130 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.db_write_buffer_size: 0 2024-07-15T06:39:21.130 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.write_buffer_manager: 0x55d79ebe87e0 2024-07-15T06:39:21.130 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.access_hint_on_compaction_start: 1 2024-07-15T06:39:21.130 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-07-15T06:39:21.130 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2024-07-15T06:39:21.130 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.use_adaptive_mutex: 0 2024-07-15T06:39:21.130 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.rate_limiter: (nil) 2024-07-15T06:39:21.131 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-07-15T06:39:21.131 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.wal_recovery_mode: 2 2024-07-15T06:39:21.131 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.enable_thread_tracking: 0 2024-07-15T06:39:21.131 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.enable_pipelined_write: 0 2024-07-15T06:39:21.131 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.unordered_write: 0 2024-07-15T06:39:21.131 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2024-07-15T06:39:21.131 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-07-15T06:39:21.131 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.write_thread_max_yield_usec: 100 2024-07-15T06:39:21.131 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2024-07-15T06:39:21.131 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.row_cache: None 2024-07-15T06:39:21.131 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.wal_filter: None 2024-07-15T06:39:21.131 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.avoid_flush_during_recovery: 0 2024-07-15T06:39:21.131 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.allow_ingest_behind: 0 2024-07-15T06:39:21.132 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.preserve_deletes: 0 2024-07-15T06:39:21.132 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.two_write_queues: 0 2024-07-15T06:39:21.132 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.manual_wal_flush: 0 2024-07-15T06:39:21.132 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.atomic_flush: 0 2024-07-15T06:39:21.132 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-07-15T06:39:21.132 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.persist_stats_to_disk: 0 2024-07-15T06:39:21.132 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.write_dbid_to_manifest: 0 2024-07-15T06:39:21.132 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.log_readahead_size: 0 2024-07-15T06:39:21.132 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.file_checksum_gen_factory: Unknown 2024-07-15T06:39:21.132 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.best_efforts_recovery: 0 2024-07-15T06:39:21.132 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-07-15T06:39:21.132 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-07-15T06:39:21.132 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.allow_data_in_errors: 0 2024-07-15T06:39:21.132 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.db_host_id: __hostname__ 2024-07-15T06:39:21.133 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.max_background_jobs: 2 2024-07-15T06:39:21.133 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.max_background_compactions: -1 2024-07-15T06:39:21.133 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.max_subcompactions: 1 2024-07-15T06:39:21.133 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2024-07-15T06:39:21.133 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-07-15T06:39:21.133 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.delayed_write_rate : 16777216 2024-07-15T06:39:21.133 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.max_total_wal_size: 0 2024-07-15T06:39:21.133 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-07-15T06:39:21.133 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.stats_dump_period_sec: 600 2024-07-15T06:39:21.133 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.stats_persist_period_sec: 600 2024-07-15T06:39:21.133 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.stats_history_buffer_size: 1048576 2024-07-15T06:39:21.133 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.max_open_files: -1 2024-07-15T06:39:21.134 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.bytes_per_sync: 0 2024-07-15T06:39:21.134 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.wal_bytes_per_sync: 0 2024-07-15T06:39:21.134 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.strict_bytes_per_sync: 0 2024-07-15T06:39:21.134 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.compaction_readahead_size: 0 2024-07-15T06:39:21.134 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.max_background_flushes: -1 2024-07-15T06:39:21.134 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Compression algorithms supported: 2024-07-15T06:39:21.134 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: kZSTD supported: 0 2024-07-15T06:39:21.134 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: kXpressCompression supported: 0 2024-07-15T06:39:21.134 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: kBZip2Compression supported: 0 2024-07-15T06:39:21.134 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: kZSTDNotFinalCompression supported: 0 2024-07-15T06:39:21.134 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: kLZ4Compression supported: 1 2024-07-15T06:39:21.134 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: kZlibCompression supported: 1 2024-07-15T06:39:21.134 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: kLZ4HCCompression supported: 1 2024-07-15T06:39:21.135 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: kSnappyCompression supported: 1 2024-07-15T06:39:21.135 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Fast CRC32 supported: Supported on x86 2024-07-15T06:39:21.135 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 2024-07-15T06:39:21.135 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2024-07-15T06:39:21.135 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-07-15T06:39:21.135 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.merge_operator: 2024-07-15T06:39:21.136 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.compaction_filter: None 2024-07-15T06:39:21.137 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.compaction_filter_factory: None 2024-07-15T06:39:21.137 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.sst_partitioner_factory: None 2024-07-15T06:39:21.137 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.memtable_factory: SkipListFactory 2024-07-15T06:39:21.137 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.table_factory: BlockBasedTable 2024-07-15T06:39:21.137 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55d79ead3a68) 2024-07-15T06:39:21.137 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: cache_index_and_filter_blocks: 1 2024-07-15T06:39:21.137 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: cache_index_and_filter_blocks_with_high_priority: 0 2024-07-15T06:39:21.137 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: pin_l0_filter_and_index_blocks_in_cache: 0 2024-07-15T06:39:21.137 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: pin_top_level_index_and_filter: 1 2024-07-15T06:39:21.137 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: index_type: 0 2024-07-15T06:39:21.137 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: data_block_index_type: 0 2024-07-15T06:39:21.137 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: index_shortening: 1 2024-07-15T06:39:21.138 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: data_block_hash_table_util_ratio: 0.750000 2024-07-15T06:39:21.138 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: hash_index_allow_collision: 1 2024-07-15T06:39:21.138 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: checksum: 1 2024-07-15T06:39:21.138 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: no_block_cache: 0 2024-07-15T06:39:21.138 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: block_cache: 0x55d79eb07350 2024-07-15T06:39:21.138 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: block_cache_name: BinnedLRUCache 2024-07-15T06:39:21.138 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: block_cache_options: 2024-07-15T06:39:21.138 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: capacity : 536870912 2024-07-15T06:39:21.138 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: num_shard_bits : 4 2024-07-15T06:39:21.138 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: strict_capacity_limit : 0 2024-07-15T06:39:21.138 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: high_pri_pool_ratio: 0.000 2024-07-15T06:39:21.138 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: block_cache_compressed: (nil) 2024-07-15T06:39:21.138 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: persistent_cache: (nil) 2024-07-15T06:39:21.139 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: block_size: 4096 2024-07-15T06:39:21.139 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: block_size_deviation: 10 2024-07-15T06:39:21.139 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: block_restart_interval: 16 2024-07-15T06:39:21.139 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: index_block_restart_interval: 1 2024-07-15T06:39:21.139 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: metadata_block_size: 4096 2024-07-15T06:39:21.139 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: partition_filters: 0 2024-07-15T06:39:21.139 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: use_delta_encoding: 1 2024-07-15T06:39:21.139 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: filter_policy: rocksdb.BuiltinBloomFilter 2024-07-15T06:39:21.139 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: whole_key_filtering: 1 2024-07-15T06:39:21.139 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: verify_compression: 0 2024-07-15T06:39:21.139 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: read_amp_bytes_per_bit: 0 2024-07-15T06:39:21.139 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: format_version: 4 2024-07-15T06:39:21.139 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: enable_index_compression: 1 2024-07-15T06:39:21.140 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: block_align: 0 2024-07-15T06:39:21.140 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.write_buffer_size: 33554432 2024-07-15T06:39:21.140 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.max_write_buffer_number: 2 2024-07-15T06:39:21.140 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.compression: NoCompression 2024-07-15T06:39:21.140 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.bottommost_compression: Disabled 2024-07-15T06:39:21.140 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.prefix_extractor: nullptr 2024-07-15T06:39:21.140 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-07-15T06:39:21.140 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.num_levels: 7 2024-07-15T06:39:21.140 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-07-15T06:39:21.140 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-07-15T06:39:21.140 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-07-15T06:39:21.140 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-07-15T06:39:21.140 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2024-07-15T06:39:21.140 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-07-15T06:39:21.141 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-07-15T06:39:21.141 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-07-15T06:39:21.141 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-07-15T06:39:21.141 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2024-07-15T06:39:21.141 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.compression_opts.window_bits: -14 2024-07-15T06:39:21.141 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.compression_opts.level: 32767 2024-07-15T06:39:21.141 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.compression_opts.strategy: 0 2024-07-15T06:39:21.141 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-07-15T06:39:21.141 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-07-15T06:39:21.141 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.compression_opts.parallel_threads: 1 2024-07-15T06:39:21.141 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.compression_opts.enabled: false 2024-07-15T06:39:21.141 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-07-15T06:39:21.141 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-07-15T06:39:21.141 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.level0_stop_writes_trigger: 36 2024-07-15T06:39:21.142 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.target_file_size_base: 67108864 2024-07-15T06:39:21.142 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.target_file_size_multiplier: 1 2024-07-15T06:39:21.142 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2024-07-15T06:39:21.142 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-07-15T06:39:21.142 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-07-15T06:39:21.142 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-07-15T06:39:21.142 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-07-15T06:39:21.142 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-07-15T06:39:21.142 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-07-15T06:39:21.142 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-07-15T06:39:21.142 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-07-15T06:39:21.142 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-07-15T06:39:21.142 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-07-15T06:39:21.143 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.max_compaction_bytes: 1677721600 2024-07-15T06:39:21.143 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.arena_block_size: 4194304 2024-07-15T06:39:21.143 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-07-15T06:39:21.143 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-07-15T06:39:21.143 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-07-15T06:39:21.143 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.disable_auto_compactions: 0 2024-07-15T06:39:21.143 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-07-15T06:39:21.143 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-07-15T06:39:21.143 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-07-15T06:39:21.143 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-07-15T06:39:21.143 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-07-15T06:39:21.143 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-07-15T06:39:21.143 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-07-15T06:39:21.144 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-07-15T06:39:21.144 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-07-15T06:39:21.144 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-07-15T06:39:21.144 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.table_properties_collectors: 2024-07-15T06:39:21.144 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.inplace_update_support: 0 2024-07-15T06:39:21.144 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.inplace_update_num_locks: 10000 2024-07-15T06:39:21.144 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-07-15T06:39:21.144 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.memtable_whole_key_filtering: 0 2024-07-15T06:39:21.144 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.memtable_huge_page_size: 0 2024-07-15T06:39:21.144 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.bloom_locality: 0 2024-07-15T06:39:21.144 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.max_successive_merges: 0 2024-07-15T06:39:21.144 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.optimize_filters_for_hits: 0 2024-07-15T06:39:21.144 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.paranoid_file_checks: 0 2024-07-15T06:39:21.144 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.force_consistency_checks: 1 2024-07-15T06:39:21.144 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.report_bg_io_stats: 0 2024-07-15T06:39:21.145 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.ttl: 2592000 2024-07-15T06:39:21.145 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.periodic_compaction_seconds: 0 2024-07-15T06:39:21.145 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.enable_blob_files: false 2024-07-15T06:39:21.145 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.min_blob_size: 0 2024-07-15T06:39:21.145 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.blob_file_size: 268435456 2024-07-15T06:39:21.145 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.blob_compression_type: NoCompression 2024-07-15T06:39:21.145 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.enable_blob_garbage_collection: false 2024-07-15T06:39:21.145 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-07-15T06:39:21.145 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-07-15T06:39:21.145 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 0 2024-07-15T06:39:21.145 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: [db/version_set.cc:4082] Creating manifest 7 2024-07-15T06:39:21.145 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1721025560886935, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-07-15T06:39:21.145 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 4 rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #4 mode 2 2024-07-15T06:39:21.146 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.882+0000 7fd48320d900 3 rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-07-15T06:39:21.146 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.886+0000 7fd48320d900 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1721025560887349, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1545, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 523, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 401, "raw_average_value_size": 80, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1721025560, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "0919f36f-af36-4d21-9a71-e516f4d1bb7b", "db_session_id": "BBRJRU8AJD2LS0T2WJXB"}} 2024-07-15T06:39:21.146 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.886+0000 7fd48320d900 4 rocksdb: [db/version_set.cc:4082] Creating manifest 9 2024-07-15T06:39:21.146 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.886+0000 7fd48320d900 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1721025560887825, "job": 1, "event": "recovery_finished"} 2024-07-15T06:39:21.146 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.886+0000 7fd48320d900 4 rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-07-15T06:39:21.146 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.886+0000 7fd48320d900 4 rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x55d79eb3c700 2024-07-15T06:39:21.147 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.886+0000 7fd48320d900 4 rocksdb: DB pointer 0x55d79eb0a000 2024-07-15T06:39:21.147 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.886+0000 7fd478fd9640 4 rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2024-07-15T06:39:21.147 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.886+0000 7fd478fd9640 4 rocksdb: [db/db_impl/db_impl.cc:903] 2024-07-15T06:39:21.147 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: ** DB Stats ** 2024-07-15T06:39:21.147 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: Uptime(secs): 0.0 total, 0.0 interval 2024-07-15T06:39:21.147 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-07-15T06:39:21.147 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-07-15T06:39:21.147 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-07-15T06:39:21.147 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-07-15T06:39:21.147 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-07-15T06:39:21.147 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-07-15T06:39:21.147 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: ** Compaction Stats [default] ** 2024-07-15T06:39:21.147 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-07-15T06:39:21.148 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-07-15T06:39:21.148 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.8 0.00 0.00 1 0.000 0 0 2024-07-15T06:39:21.148 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.8 0.00 0.00 1 0.000 0 0 2024-07-15T06:39:21.148 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.8 0.00 0.00 1 0.000 0 0 2024-07-15T06:39:21.148 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: ** Compaction Stats [default] ** 2024-07-15T06:39:21.148 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-07-15T06:39:21.148 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-07-15T06:39:21.148 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 3.8 0.00 0.00 1 0.000 0 0 2024-07-15T06:39:21.148 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: Uptime(secs): 0.0 total, 0.0 interval 2024-07-15T06:39:21.148 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: Flush(GB): cumulative 0.000, interval 0.000 2024-07-15T06:39:21.148 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: AddFile(GB): cumulative 0.000, interval 0.000 2024-07-15T06:39:21.148 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: AddFile(Total Files): cumulative 0, interval 0 2024-07-15T06:39:21.148 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: AddFile(L0 Files): cumulative 0, interval 0 2024-07-15T06:39:21.148 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: AddFile(Keys): cumulative 0, interval 0 2024-07-15T06:39:21.149 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: Cumulative compaction: 0.00 GB write, 0.42 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-07-15T06:39:21.149 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: Interval compaction: 0.00 GB write, 0.42 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-07-15T06:39:21.149 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-07-15T06:39:21.149 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: ** File Read Latency Histogram By Level [default] ** 2024-07-15T06:39:21.149 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: ** Compaction Stats [default] ** 2024-07-15T06:39:21.149 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-07-15T06:39:21.149 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-07-15T06:39:21.149 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.8 0.00 0.00 1 0.000 0 0 2024-07-15T06:39:21.149 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.8 0.00 0.00 1 0.000 0 0 2024-07-15T06:39:21.149 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-07-15T06:39:21.149 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: ** Compaction Stats [default] ** 2024-07-15T06:39:21.149 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-07-15T06:39:21.149 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-07-15T06:39:21.149 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 3.8 0.00 0.00 1 0.000 0 0 2024-07-15T06:39:21.150 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: Uptime(secs): 0.0 total, 0.0 interval 2024-07-15T06:39:21.150 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: Flush(GB): cumulative 0.000, interval 0.000 2024-07-15T06:39:21.150 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: AddFile(GB): cumulative 0.000, interval 0.000 2024-07-15T06:39:21.150 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: AddFile(Total Files): cumulative 0, interval 0 2024-07-15T06:39:21.150 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: AddFile(L0 Files): cumulative 0, interval 0 2024-07-15T06:39:21.150 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: AddFile(Keys): cumulative 0, interval 0 2024-07-15T06:39:21.150 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: Cumulative compaction: 0.00 GB write, 0.41 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-07-15T06:39:21.150 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-07-15T06:39:21.150 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-07-15T06:39:21.150 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: ** File Read Latency Histogram By Level [default] ** 2024-07-15T06:39:21.150 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.886+0000 7fd48320d900 0 mon.c does not exist in monmap, will attempt to join an existing cluster 2024-07-15T06:39:21.151 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.886+0000 7fd48320d900 0 using public_addrv [v2:172.21.15.89:3301/0,v1:172.21.15.89:6790/0] 2024-07-15T06:39:21.151 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.886+0000 7fd48320d900 0 starting mon.c rank -1 at public addrs [v2:172.21.15.89:3301/0,v1:172.21.15.89:6790/0] at bind addrs [v2:172.21.15.89:3301/0,v1:172.21.15.89:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid 75ba02bc-4274-11ef-bcad-c7b262605968 2024-07-15T06:39:21.151 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.886+0000 7fd48320d900 1 mon.c@-1(???) e0 preinit fsid 75ba02bc-4274-11ef-bcad-c7b262605968 2024-07-15T06:39:21.151 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.910+0000 7fd47bfdf640 0 mon.c@-1(synchronizing).mds e1 new map 2024-07-15T06:39:21.151 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.910+0000 7fd47bfdf640 0 mon.c@-1(synchronizing).mds e1 print_map 2024-07-15T06:39:21.151 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: e1 2024-07-15T06:39:21.151 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: enable_multiple, ever_enabled_multiple: 1,1 2024-07-15T06:39:21.151 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-07-15T06:39:21.151 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: legacy client fscid: -1 2024-07-15T06:39:21.151 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: 2024-07-15T06:39:21.151 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: No filesystems configured 2024-07-15T06:39:21.151 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.910+0000 7fd47bfdf640 1 mon.c@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-07-15T06:39:21.151 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.910+0000 7fd47bfdf640 1 mon.c@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-07-15T06:39:21.152 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.910+0000 7fd47bfdf640 1 mon.c@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-07-15T06:39:21.152 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.910+0000 7fd47bfdf640 1 mon.c@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-07-15T06:39:21.152 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.910+0000 7fd47bfdf640 1 mon.c@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-07-15T06:39:21.152 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.910+0000 7fd47bfdf640 1 mon.c@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-07-15T06:39:21.152 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.910+0000 7fd47bfdf640 0 mon.c@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2024-07-15T06:39:21.152 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.910+0000 7fd47bfdf640 0 mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-07-15T06:39:21.152 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.910+0000 7fd47bfdf640 0 mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-07-15T06:39:21.152 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.910+0000 7fd47bfdf640 0 mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-07-15T06:39:21.152 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: cephadm 2024-07-15T06:38:36.479629+0000 mgr.y (mgr.14150) 17 : cephadm [INF] Updating smithi089:/etc/ceph/ceph.client.admin.keyring 2024-07-15T06:39:21.152 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: cephadm 2024-07-15T06:38:36.560182+0000 mgr.y (mgr.14150) 18 : cephadm [INF] Updating smithi089:/var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/config/ceph.client.admin.keyring 2024-07-15T06:39:21.152 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: cluster 2024-07-15T06:38:36.857875+0000 mgr.y (mgr.14150) 19 : cluster [DBG] pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:21.152 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: cluster 2024-07-15T06:38:38.858285+0000 mgr.y (mgr.14150) 20 : cluster [DBG] pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:21.152 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: cluster 2024-07-15T06:38:40.858680+0000 mgr.y (mgr.14150) 21 : cluster [DBG] pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:21.153 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: cluster 2024-07-15T06:38:42.859142+0000 mgr.y (mgr.14150) 22 : cluster [DBG] pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:21.153 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: audit 2024-07-15T06:38:44.582460+0000 mgr.y (mgr.14150) 23 : audit [DBG] from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi134", "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T06:39:21.153 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: cluster 2024-07-15T06:38:44.859531+0000 mgr.y (mgr.14150) 24 : cluster [DBG] pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:21.153 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: cephadm 2024-07-15T06:38:45.631430+0000 mgr.y (mgr.14150) 25 : cephadm [INF] Deploying cephadm binary to smithi134 2024-07-15T06:39:21.153 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: cluster 2024-07-15T06:38:46.859819+0000 mgr.y (mgr.14150) 26 : cluster [DBG] pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:21.153 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: audit 2024-07-15T06:38:47.869432+0000 mon.a (mon.0) 110 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:21.153 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: audit 2024-07-15T06:38:47.870968+0000 mon.a (mon.0) 111 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:39:21.153 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: audit 2024-07-15T06:38:48.113717+0000 mon.a (mon.0) 112 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:21.153 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: cephadm 2024-07-15T06:38:47.870230+0000 mgr.y (mgr.14150) 27 : cephadm [INF] Added host smithi134 2024-07-15T06:39:21.153 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: cluster 2024-07-15T06:38:48.860151+0000 mgr.y (mgr.14150) 28 : cluster [DBG] pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:21.153 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: audit 2024-07-15T06:38:49.378552+0000 mon.a (mon.0) 113 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:21.153 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: audit 2024-07-15T06:38:50.067812+0000 mon.a (mon.0) 114 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:21.153 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: cluster 2024-07-15T06:38:50.860586+0000 mgr.y (mgr.14150) 29 : cluster [DBG] pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:21.154 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: cluster 2024-07-15T06:38:52.860997+0000 mgr.y (mgr.14150) 30 : cluster [DBG] pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:21.154 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: cluster 2024-07-15T06:38:54.861476+0000 mgr.y (mgr.14150) 31 : cluster [DBG] pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:21.154 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: audit 2024-07-15T06:38:55.631996+0000 mon.a (mon.0) 115 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:21.154 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: audit 2024-07-15T06:38:55.639547+0000 mon.a (mon.0) 116 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:21.154 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: audit 2024-07-15T06:38:55.648078+0000 mon.a (mon.0) 117 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:21.154 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: audit 2024-07-15T06:38:55.655792+0000 mon.a (mon.0) 118 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:21.154 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: audit 2024-07-15T06:38:55.657223+0000 mon.a (mon.0) 119 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi134", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:39:21.154 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: audit 2024-07-15T06:38:55.659171+0000 mon.a (mon.0) 120 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:39:21.154 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: audit 2024-07-15T06:38:55.660786+0000 mon.a (mon.0) 121 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:39:21.154 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: audit 2024-07-15T06:38:56.171394+0000 mon.a (mon.0) 122 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:21.154 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: audit 2024-07-15T06:38:56.178840+0000 mon.a (mon.0) 123 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:21.154 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: audit 2024-07-15T06:38:56.187137+0000 mon.a (mon.0) 124 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:21.154 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: cephadm 2024-07-15T06:38:55.663031+0000 mgr.y (mgr.14150) 32 : cephadm [INF] Updating smithi134:/etc/ceph/ceph.conf 2024-07-15T06:39:21.155 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: cephadm 2024-07-15T06:38:55.817434+0000 mgr.y (mgr.14150) 33 : cephadm [INF] Updating smithi134:/var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/config/ceph.conf 2024-07-15T06:39:21.155 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: cephadm 2024-07-15T06:38:55.927217+0000 mgr.y (mgr.14150) 34 : cephadm [INF] Updating smithi134:/etc/ceph/ceph.client.admin.keyring 2024-07-15T06:39:21.155 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: cephadm 2024-07-15T06:38:56.050473+0000 mgr.y (mgr.14150) 35 : cephadm [INF] Updating smithi134:/var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/config/ceph.client.admin.keyring 2024-07-15T06:39:21.155 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: audit 2024-07-15T06:38:56.194570+0000 mgr.y (mgr.14150) 36 : audit [DBG] from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-07-15T06:39:21.155 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: cluster 2024-07-15T06:38:56.861904+0000 mgr.y (mgr.14150) 37 : cluster [DBG] pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:21.155 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: cluster 2024-07-15T06:38:58.862306+0000 mgr.y (mgr.14150) 38 : cluster [DBG] pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:21.155 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: cluster 2024-07-15T06:39:00.862717+0000 mgr.y (mgr.14150) 39 : cluster [DBG] pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:21.155 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: cluster 2024-07-15T06:39:02.863123+0000 mgr.y (mgr.14150) 40 : cluster [DBG] pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:21.155 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: audit 2024-07-15T06:39:04.402983+0000 mon.a (mon.0) 125 : audit [INF] from='client.? 172.21.15.89:0/2297113159' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-07-15T06:39:21.155 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: cluster 2024-07-15T06:39:04.863556+0000 mgr.y (mgr.14150) 41 : cluster [DBG] pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:21.155 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: audit 2024-07-15T06:39:05.408403+0000 mon.a (mon.0) 126 : audit [INF] from='client.? 172.21.15.89:0/2297113159' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-07-15T06:39:21.155 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: cluster 2024-07-15T06:39:05.408553+0000 mon.a (mon.0) 127 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2024-07-15T06:39:21.156 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: cluster 2024-07-15T06:39:06.863953+0000 mgr.y (mgr.14150) 42 : cluster [DBG] pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:21.156 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: cluster 2024-07-15T06:39:08.864362+0000 mgr.y (mgr.14150) 43 : cluster [DBG] pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:21.156 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: audit 2024-07-15T06:39:09.958010+0000 mon.a (mon.0) 128 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:21.156 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: audit 2024-07-15T06:39:09.959579+0000 mon.a (mon.0) 129 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:39:21.156 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: audit 2024-07-15T06:39:09.963084+0000 mon.a (mon.0) 130 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:39:21.156 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: audit 2024-07-15T06:39:09.964857+0000 mon.a (mon.0) 131 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:39:21.156 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: audit 2024-07-15T06:39:09.972303+0000 mon.a (mon.0) 132 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:21.156 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: audit 2024-07-15T06:39:09.976069+0000 mon.a (mon.0) 133 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-07-15T06:39:21.156 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: audit 2024-07-15T06:39:09.977708+0000 mon.a (mon.0) 134 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:39:21.156 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: audit 2024-07-15T06:39:09.947711+0000 mgr.y (mgr.14150) 44 : audit [DBG] from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi089:172.21.15.89=a;smithi089:[v2:172.21.15.89:3301,v1:172.21.15.89:6790]=c;smithi134:172.21.15.134=b", "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T06:39:21.156 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: cephadm 2024-07-15T06:39:09.953036+0000 mgr.y (mgr.14150) 45 : cephadm [INF] Saving service mon spec with placement smithi089:172.21.15.89=a;smithi089:[v2:172.21.15.89:3301,v1:172.21.15.89:6790]=c;smithi134:172.21.15.134=b;count:3 2024-07-15T06:39:21.156 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: cephadm 2024-07-15T06:39:09.979604+0000 mgr.y (mgr.14150) 46 : cephadm [INF] Deploying daemon mon.b on smithi134 2024-07-15T06:39:21.156 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: cluster 2024-07-15T06:39:10.864877+0000 mgr.y (mgr.14150) 47 : cluster [DBG] pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:21.157 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: cluster 2024-07-15T06:39:12.865400+0000 mgr.y (mgr.14150) 48 : cluster [DBG] pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:21.157 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: cluster 2024-07-15T06:39:14.865862+0000 mgr.y (mgr.14150) 49 : cluster [DBG] pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:21.157 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: audit 2024-07-15T06:39:15.714323+0000 mon.a (mon.0) 135 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:21.157 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: audit 2024-07-15T06:39:15.719717+0000 mon.a (mon.0) 136 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:21.157 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: audit 2024-07-15T06:39:15.728023+0000 mon.a (mon.0) 137 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:21.157 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: audit 2024-07-15T06:39:15.729261+0000 mon.a (mon.0) 138 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-07-15T06:39:21.157 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: audit 2024-07-15T06:39:15.730800+0000 mon.a (mon.0) 139 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:39:21.157 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: audit 2024-07-15T06:39:15.832631+0000 mon.a (mon.0) 140 : audit [DBG] from='client.? 172.21.15.134:0/953884829' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-07-15T06:39:21.157 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:20 smithi089 bash[31372]: debug 2024-07-15T06:39:20.914+0000 7fd47bfdf640 1 mon.c@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2024-07-15T06:39:21.522 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.b/config 2024-07-15T06:39:22.560 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:22 smithi134 bash[24068]: cluster 2024-07-15T06:39:16.866348+0000 mgr.y (mgr.14150) 51 : cluster [DBG] pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:22.560 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:22 smithi134 bash[24068]: audit 2024-07-15T06:39:17.085346+0000 mon.a (mon.0) 142 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-07-15T06:39:22.560 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:22 smithi134 bash[24068]: cluster 2024-07-15T06:39:17.085513+0000 mon.a (mon.0) 143 : cluster [INF] mon.a calling monitor election 2024-07-15T06:39:22.560 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:22 smithi134 bash[24068]: audit 2024-07-15T06:39:17.107880+0000 mon.a (mon.0) 144 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-15T06:39:22.560 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:22 smithi134 bash[24068]: audit 2024-07-15T06:39:18.083034+0000 mon.a (mon.0) 145 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-15T06:39:22.560 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:22 smithi134 bash[24068]: cluster 2024-07-15T06:39:18.866805+0000 mgr.y (mgr.14150) 52 : cluster [DBG] pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:22.560 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:22 smithi134 bash[24068]: audit 2024-07-15T06:39:19.083498+0000 mon.a (mon.0) 146 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-15T06:39:22.560 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:22 smithi134 bash[24068]: cluster 2024-07-15T06:39:19.085406+0000 mon.b (mon.1) 1 : cluster [INF] mon.b calling monitor election 2024-07-15T06:39:22.560 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:22 smithi134 bash[24068]: audit 2024-07-15T06:39:20.083759+0000 mon.a (mon.0) 147 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-15T06:39:22.561 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:22 smithi134 bash[24068]: cluster 2024-07-15T06:39:20.867107+0000 mgr.y (mgr.14150) 53 : cluster [DBG] pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:22.561 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:22 smithi134 bash[24068]: audit 2024-07-15T06:39:20.920282+0000 mon.a (mon.0) 148 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-15T06:39:22.561 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:22 smithi134 bash[24068]: audit 2024-07-15T06:39:21.083966+0000 mon.a (mon.0) 149 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-15T06:39:22.561 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:22 smithi134 bash[24068]: audit 2024-07-15T06:39:21.921165+0000 mon.a (mon.0) 150 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-15T06:39:22.561 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:22 smithi134 bash[24068]: audit 2024-07-15T06:39:22.084023+0000 mon.a (mon.0) 151 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-15T06:39:22.561 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:22 smithi134 bash[24068]: cluster 2024-07-15T06:39:22.131161+0000 mon.a (mon.0) 152 : cluster [INF] mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-07-15T06:39:22.561 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:22 smithi134 bash[24068]: cluster 2024-07-15T06:39:22.136981+0000 mon.a (mon.0) 153 : cluster [DBG] monmap e2: 2 mons at {a=[v2:172.21.15.89:3300/0,v1:172.21.15.89:6789/0],b=[v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-07-15T06:39:22.561 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:22 smithi134 bash[24068]: cluster 2024-07-15T06:39:22.137986+0000 mon.a (mon.0) 154 : cluster [DBG] fsmap 2024-07-15T06:39:22.561 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:22 smithi134 bash[24068]: cluster 2024-07-15T06:39:22.138039+0000 mon.a (mon.0) 155 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2024-07-15T06:39:22.561 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:22 smithi134 bash[24068]: cluster 2024-07-15T06:39:22.138276+0000 mon.a (mon.0) 156 : cluster [DBG] mgrmap e13: y(active, since 75s) 2024-07-15T06:39:22.561 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:22 smithi134 bash[24068]: cluster 2024-07-15T06:39:22.138508+0000 mon.a (mon.0) 157 : cluster [INF] overall HEALTH_OK 2024-07-15T06:39:22.561 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:22 smithi134 bash[24068]: audit 2024-07-15T06:39:22.144615+0000 mon.a (mon.0) 158 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:22.562 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:22 smithi134 bash[24068]: audit 2024-07-15T06:39:22.151833+0000 mon.a (mon.0) 159 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:22.562 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:22 smithi134 bash[24068]: audit 2024-07-15T06:39:22.160003+0000 mon.a (mon.0) 160 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:22.562 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:22 smithi134 bash[24068]: audit 2024-07-15T06:39:22.167109+0000 mon.a (mon.0) 161 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:22.625 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:22 smithi089 bash[21793]: cluster 2024-07-15T06:39:16.866348+0000 mgr.y (mgr.14150) 51 : cluster [DBG] pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:22.625 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:22 smithi089 bash[21793]: audit 2024-07-15T06:39:17.085346+0000 mon.a (mon.0) 142 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-07-15T06:39:22.626 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:22 smithi089 bash[21793]: cluster 2024-07-15T06:39:17.085513+0000 mon.a (mon.0) 143 : cluster [INF] mon.a calling monitor election 2024-07-15T06:39:22.626 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:22 smithi089 bash[21793]: audit 2024-07-15T06:39:17.107880+0000 mon.a (mon.0) 144 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-15T06:39:22.626 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:22 smithi089 bash[21793]: audit 2024-07-15T06:39:18.083034+0000 mon.a (mon.0) 145 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-15T06:39:22.626 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:22 smithi089 bash[21793]: cluster 2024-07-15T06:39:18.866805+0000 mgr.y (mgr.14150) 52 : cluster [DBG] pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:22.626 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:22 smithi089 bash[21793]: audit 2024-07-15T06:39:19.083498+0000 mon.a (mon.0) 146 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-15T06:39:22.626 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:22 smithi089 bash[21793]: cluster 2024-07-15T06:39:19.085406+0000 mon.b (mon.1) 1 : cluster [INF] mon.b calling monitor election 2024-07-15T06:39:22.626 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:22 smithi089 bash[21793]: audit 2024-07-15T06:39:20.083759+0000 mon.a (mon.0) 147 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-15T06:39:22.626 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:22 smithi089 bash[21793]: cluster 2024-07-15T06:39:20.867107+0000 mgr.y (mgr.14150) 53 : cluster [DBG] pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:22.626 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:22 smithi089 bash[21793]: audit 2024-07-15T06:39:20.920282+0000 mon.a (mon.0) 148 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-15T06:39:22.626 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:22 smithi089 bash[21793]: audit 2024-07-15T06:39:21.083966+0000 mon.a (mon.0) 149 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-15T06:39:22.626 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:22 smithi089 bash[21793]: audit 2024-07-15T06:39:21.921165+0000 mon.a (mon.0) 150 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-15T06:39:22.626 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:22 smithi089 bash[21793]: audit 2024-07-15T06:39:22.084023+0000 mon.a (mon.0) 151 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-15T06:39:22.627 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:22 smithi089 bash[21793]: cluster 2024-07-15T06:39:22.131161+0000 mon.a (mon.0) 152 : cluster [INF] mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-07-15T06:39:22.627 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:22 smithi089 bash[21793]: cluster 2024-07-15T06:39:22.136981+0000 mon.a (mon.0) 153 : cluster [DBG] monmap e2: 2 mons at {a=[v2:172.21.15.89:3300/0,v1:172.21.15.89:6789/0],b=[v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-07-15T06:39:22.627 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:22 smithi089 bash[21793]: cluster 2024-07-15T06:39:22.137986+0000 mon.a (mon.0) 154 : cluster [DBG] fsmap 2024-07-15T06:39:22.627 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:22 smithi089 bash[21793]: cluster 2024-07-15T06:39:22.138039+0000 mon.a (mon.0) 155 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2024-07-15T06:39:22.627 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:22 smithi089 bash[21793]: cluster 2024-07-15T06:39:22.138276+0000 mon.a (mon.0) 156 : cluster [DBG] mgrmap e13: y(active, since 75s) 2024-07-15T06:39:22.627 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:22 smithi089 bash[21793]: cluster 2024-07-15T06:39:22.138508+0000 mon.a (mon.0) 157 : cluster [INF] overall HEALTH_OK 2024-07-15T06:39:22.627 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:22 smithi089 bash[21793]: audit 2024-07-15T06:39:22.144615+0000 mon.a (mon.0) 158 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:22.627 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:22 smithi089 bash[21793]: audit 2024-07-15T06:39:22.151833+0000 mon.a (mon.0) 159 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:22.627 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:22 smithi089 bash[21793]: audit 2024-07-15T06:39:22.160003+0000 mon.a (mon.0) 160 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:22.627 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:22 smithi089 bash[21793]: audit 2024-07-15T06:39:22.167109+0000 mon.a (mon.0) 161 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:23.374 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:39:23 smithi089 bash[22045]: debug 2024-07-15T06:39:23.082+0000 7ff816c49640 -1 mgr.server handle_report got status from non-daemon mon.b 2024-07-15T06:39:28.275 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-07-15T06:39:28.276 INFO:teuthology.orchestra.run.smithi134.stdout:{"epoch":3,"fsid":"75ba02bc-4274-11ef-bcad-c7b262605968","modified":"2024-07-15T06:39:22.921843Z","created":"2024-07-15T06:37:05.016290Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:3300","nonce":0},{"type":"v1","addr":"172.21.15.89:6789","nonce":0}]},"addr":"172.21.15.89:6789/0","public_addr":"172.21.15.89:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:3300","nonce":0},{"type":"v1","addr":"172.21.15.134:6789","nonce":0}]},"addr":"172.21.15.134:6789/0","public_addr":"172.21.15.134:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:3301","nonce":0},{"type":"v1","addr":"172.21.15.89:6790","nonce":0}]},"addr":"172.21.15.89:6790/0","public_addr":"172.21.15.89:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1,2]} 2024-07-15T06:39:28.276 INFO:teuthology.orchestra.run.smithi134.stderr:dumped monmap epoch 3 2024-07-15T06:39:28.310 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:28 smithi134 bash[24068]: cluster 2024-07-15T06:39:22.867502+0000 mgr.y (mgr.14150) 54 : cluster [DBG] pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:28.310 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:28 smithi134 bash[24068]: audit 2024-07-15T06:39:22.927187+0000 mon.a (mon.0) 164 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-07-15T06:39:28.310 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:28 smithi134 bash[24068]: audit 2024-07-15T06:39:22.927666+0000 mon.a (mon.0) 165 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-15T06:39:28.310 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:28 smithi134 bash[24068]: cluster 2024-07-15T06:39:22.927884+0000 mon.a (mon.0) 166 : cluster [INF] mon.a calling monitor election 2024-07-15T06:39:28.311 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:28 smithi134 bash[24068]: cluster 2024-07-15T06:39:22.930951+0000 mon.b (mon.1) 2 : cluster [INF] mon.b calling monitor election 2024-07-15T06:39:28.311 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:28 smithi134 bash[24068]: audit 2024-07-15T06:39:22.949152+0000 mon.a (mon.0) 167 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-15T06:39:28.311 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:28 smithi134 bash[24068]: audit 2024-07-15T06:39:23.921644+0000 mon.a (mon.0) 168 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-15T06:39:28.311 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:28 smithi134 bash[24068]: cluster 2024-07-15T06:39:24.867894+0000 mgr.y (mgr.14150) 55 : cluster [DBG] pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:28.311 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:28 smithi134 bash[24068]: audit 2024-07-15T06:39:24.921910+0000 mon.a (mon.0) 169 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-15T06:39:28.311 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:28 smithi134 bash[24068]: cluster 2024-07-15T06:39:24.923960+0000 mon.c (mon.2) 1 : cluster [INF] mon.c calling monitor election 2024-07-15T06:39:28.311 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:28 smithi134 bash[24068]: audit 2024-07-15T06:39:25.921719+0000 mon.a (mon.0) 170 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-15T06:39:28.312 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:28 smithi134 bash[24068]: cluster 2024-07-15T06:39:26.868289+0000 mgr.y (mgr.14150) 56 : cluster [DBG] pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:28.312 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:28 smithi134 bash[24068]: audit 2024-07-15T06:39:26.922033+0000 mon.a (mon.0) 171 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-15T06:39:28.312 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:28 smithi134 bash[24068]: audit 2024-07-15T06:39:27.922173+0000 mon.a (mon.0) 172 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-15T06:39:28.312 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:28 smithi134 bash[24068]: cluster 2024-07-15T06:39:27.933852+0000 mon.a (mon.0) 173 : cluster [INF] mon.a calling monitor election 2024-07-15T06:39:28.312 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:28 smithi134 bash[24068]: cluster 2024-07-15T06:39:27.975155+0000 mon.a (mon.0) 174 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-07-15T06:39:28.312 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:28 smithi134 bash[24068]: cluster 2024-07-15T06:39:27.983988+0000 mon.a (mon.0) 175 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.89:3300/0,v1:172.21.15.89:6789/0],b=[v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0],c=[v2:172.21.15.89:3301/0,v1:172.21.15.89:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-07-15T06:39:28.312 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:28 smithi134 bash[24068]: cluster 2024-07-15T06:39:27.985027+0000 mon.a (mon.0) 176 : cluster [DBG] fsmap 2024-07-15T06:39:28.312 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:28 smithi134 bash[24068]: cluster 2024-07-15T06:39:27.985089+0000 mon.a (mon.0) 177 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2024-07-15T06:39:28.313 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:28 smithi134 bash[24068]: cluster 2024-07-15T06:39:27.985348+0000 mon.a (mon.0) 178 : cluster [DBG] mgrmap e13: y(active, since 81s) 2024-07-15T06:39:28.313 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:28 smithi134 bash[24068]: cluster 2024-07-15T06:39:27.985594+0000 mon.a (mon.0) 179 : cluster [INF] overall HEALTH_OK 2024-07-15T06:39:28.313 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:28 smithi134 bash[24068]: audit 2024-07-15T06:39:27.995090+0000 mon.a (mon.0) 180 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:28.313 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:28 smithi134 bash[24068]: audit 2024-07-15T06:39:28.000875+0000 mon.a (mon.0) 181 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:28.313 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:28 smithi134 bash[24068]: audit 2024-07-15T06:39:28.006581+0000 mon.a (mon.0) 182 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:28.313 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:28 smithi134 bash[24068]: audit 2024-07-15T06:39:28.012287+0000 mon.a (mon.0) 183 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:28.313 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:28 smithi134 bash[24068]: audit 2024-07-15T06:39:28.018016+0000 mon.a (mon.0) 184 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:28.375 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:28 smithi089 bash[21793]: cluster 2024-07-15T06:39:22.867502+0000 mgr.y (mgr.14150) 54 : cluster [DBG] pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:28.375 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:28 smithi089 bash[21793]: audit 2024-07-15T06:39:22.927187+0000 mon.a (mon.0) 164 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-07-15T06:39:28.375 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:28 smithi089 bash[21793]: audit 2024-07-15T06:39:22.927666+0000 mon.a (mon.0) 165 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-15T06:39:28.375 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:28 smithi089 bash[21793]: cluster 2024-07-15T06:39:22.927884+0000 mon.a (mon.0) 166 : cluster [INF] mon.a calling monitor election 2024-07-15T06:39:28.375 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:28 smithi089 bash[21793]: cluster 2024-07-15T06:39:22.930951+0000 mon.b (mon.1) 2 : cluster [INF] mon.b calling monitor election 2024-07-15T06:39:28.375 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:28 smithi089 bash[21793]: audit 2024-07-15T06:39:22.949152+0000 mon.a (mon.0) 167 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-15T06:39:28.375 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:28 smithi089 bash[21793]: audit 2024-07-15T06:39:23.921644+0000 mon.a (mon.0) 168 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-15T06:39:28.376 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:28 smithi089 bash[21793]: cluster 2024-07-15T06:39:24.867894+0000 mgr.y (mgr.14150) 55 : cluster [DBG] pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:28.376 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:28 smithi089 bash[21793]: audit 2024-07-15T06:39:24.921910+0000 mon.a (mon.0) 169 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-15T06:39:28.376 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:28 smithi089 bash[21793]: cluster 2024-07-15T06:39:24.923960+0000 mon.c (mon.2) 1 : cluster [INF] mon.c calling monitor election 2024-07-15T06:39:28.376 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:28 smithi089 bash[21793]: audit 2024-07-15T06:39:25.921719+0000 mon.a (mon.0) 170 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-15T06:39:28.376 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:28 smithi089 bash[21793]: cluster 2024-07-15T06:39:26.868289+0000 mgr.y (mgr.14150) 56 : cluster [DBG] pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:28.376 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:28 smithi089 bash[21793]: audit 2024-07-15T06:39:26.922033+0000 mon.a (mon.0) 171 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-15T06:39:28.376 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:28 smithi089 bash[21793]: audit 2024-07-15T06:39:27.922173+0000 mon.a (mon.0) 172 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-15T06:39:28.376 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:28 smithi089 bash[21793]: cluster 2024-07-15T06:39:27.933852+0000 mon.a (mon.0) 173 : cluster [INF] mon.a calling monitor election 2024-07-15T06:39:28.376 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:28 smithi089 bash[21793]: cluster 2024-07-15T06:39:27.975155+0000 mon.a (mon.0) 174 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-07-15T06:39:28.376 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:28 smithi089 bash[21793]: cluster 2024-07-15T06:39:27.983988+0000 mon.a (mon.0) 175 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.89:3300/0,v1:172.21.15.89:6789/0],b=[v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0],c=[v2:172.21.15.89:3301/0,v1:172.21.15.89:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-07-15T06:39:28.376 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:28 smithi089 bash[21793]: cluster 2024-07-15T06:39:27.985027+0000 mon.a (mon.0) 176 : cluster [DBG] fsmap 2024-07-15T06:39:28.376 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:28 smithi089 bash[21793]: cluster 2024-07-15T06:39:27.985089+0000 mon.a (mon.0) 177 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2024-07-15T06:39:28.376 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:28 smithi089 bash[21793]: cluster 2024-07-15T06:39:27.985348+0000 mon.a (mon.0) 178 : cluster [DBG] mgrmap e13: y(active, since 81s) 2024-07-15T06:39:28.377 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:28 smithi089 bash[21793]: cluster 2024-07-15T06:39:27.985594+0000 mon.a (mon.0) 179 : cluster [INF] overall HEALTH_OK 2024-07-15T06:39:28.377 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:28 smithi089 bash[21793]: audit 2024-07-15T06:39:27.995090+0000 mon.a (mon.0) 180 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:28.377 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:28 smithi089 bash[21793]: audit 2024-07-15T06:39:28.000875+0000 mon.a (mon.0) 181 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:28.377 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:28 smithi089 bash[21793]: audit 2024-07-15T06:39:28.006581+0000 mon.a (mon.0) 182 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:28.377 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:28 smithi089 bash[21793]: audit 2024-07-15T06:39:28.012287+0000 mon.a (mon.0) 183 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:28.377 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:28 smithi089 bash[21793]: audit 2024-07-15T06:39:28.018016+0000 mon.a (mon.0) 184 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:28.378 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:27 smithi089 bash[31372]: cluster 2024-07-15T06:39:16.866348+0000 mgr.y (mgr.14150) 51 : cluster [DBG] pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:28.378 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:27 smithi089 bash[31372]: audit 2024-07-15T06:39:17.085346+0000 mon.a (mon.0) 142 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-07-15T06:39:28.378 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:27 smithi089 bash[31372]: cluster 2024-07-15T06:39:17.085513+0000 mon.a (mon.0) 143 : cluster [INF] mon.a calling monitor election 2024-07-15T06:39:28.378 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:27 smithi089 bash[31372]: audit 2024-07-15T06:39:17.107880+0000 mon.a (mon.0) 144 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-15T06:39:28.378 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:27 smithi089 bash[31372]: audit 2024-07-15T06:39:18.083034+0000 mon.a (mon.0) 145 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-15T06:39:28.378 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:27 smithi089 bash[31372]: cluster 2024-07-15T06:39:18.866805+0000 mgr.y (mgr.14150) 52 : cluster [DBG] pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:28.378 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:27 smithi089 bash[31372]: audit 2024-07-15T06:39:19.083498+0000 mon.a (mon.0) 146 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-15T06:39:28.379 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:27 smithi089 bash[31372]: cluster 2024-07-15T06:39:19.085406+0000 mon.b (mon.1) 1 : cluster [INF] mon.b calling monitor election 2024-07-15T06:39:28.379 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:27 smithi089 bash[31372]: audit 2024-07-15T06:39:20.083759+0000 mon.a (mon.0) 147 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-15T06:39:28.379 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:27 smithi089 bash[31372]: cluster 2024-07-15T06:39:20.867107+0000 mgr.y (mgr.14150) 53 : cluster [DBG] pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:28.379 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:27 smithi089 bash[31372]: audit 2024-07-15T06:39:20.920282+0000 mon.a (mon.0) 148 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-15T06:39:28.379 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:27 smithi089 bash[31372]: audit 2024-07-15T06:39:21.083966+0000 mon.a (mon.0) 149 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-15T06:39:28.379 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:27 smithi089 bash[31372]: audit 2024-07-15T06:39:21.921165+0000 mon.a (mon.0) 150 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-15T06:39:28.379 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:27 smithi089 bash[31372]: audit 2024-07-15T06:39:22.084023+0000 mon.a (mon.0) 151 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-15T06:39:28.379 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:27 smithi089 bash[31372]: cluster 2024-07-15T06:39:22.131161+0000 mon.a (mon.0) 152 : cluster [INF] mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-07-15T06:39:28.379 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:27 smithi089 bash[31372]: cluster 2024-07-15T06:39:22.136981+0000 mon.a (mon.0) 153 : cluster [DBG] monmap e2: 2 mons at {a=[v2:172.21.15.89:3300/0,v1:172.21.15.89:6789/0],b=[v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-07-15T06:39:28.379 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:27 smithi089 bash[31372]: cluster 2024-07-15T06:39:22.137986+0000 mon.a (mon.0) 154 : cluster [DBG] fsmap 2024-07-15T06:39:28.379 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:27 smithi089 bash[31372]: cluster 2024-07-15T06:39:22.138039+0000 mon.a (mon.0) 155 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2024-07-15T06:39:28.379 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:27 smithi089 bash[31372]: cluster 2024-07-15T06:39:22.138276+0000 mon.a (mon.0) 156 : cluster [DBG] mgrmap e13: y(active, since 75s) 2024-07-15T06:39:28.379 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:27 smithi089 bash[31372]: cluster 2024-07-15T06:39:22.138508+0000 mon.a (mon.0) 157 : cluster [INF] overall HEALTH_OK 2024-07-15T06:39:28.380 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:27 smithi089 bash[31372]: audit 2024-07-15T06:39:22.144615+0000 mon.a (mon.0) 158 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:28.380 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:27 smithi089 bash[31372]: audit 2024-07-15T06:39:22.151833+0000 mon.a (mon.0) 159 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:28.380 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:27 smithi089 bash[31372]: audit 2024-07-15T06:39:22.160003+0000 mon.a (mon.0) 160 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:28.380 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:27 smithi089 bash[31372]: audit 2024-07-15T06:39:22.167109+0000 mon.a (mon.0) 161 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:28.380 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:28 smithi089 bash[31372]: cluster 2024-07-15T06:39:22.867502+0000 mgr.y (mgr.14150) 54 : cluster [DBG] pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:28.380 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:28 smithi089 bash[31372]: audit 2024-07-15T06:39:22.927187+0000 mon.a (mon.0) 164 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-07-15T06:39:28.380 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:28 smithi089 bash[31372]: audit 2024-07-15T06:39:22.927666+0000 mon.a (mon.0) 165 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-15T06:39:28.380 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:28 smithi089 bash[31372]: cluster 2024-07-15T06:39:22.927884+0000 mon.a (mon.0) 166 : cluster [INF] mon.a calling monitor election 2024-07-15T06:39:28.380 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:28 smithi089 bash[31372]: cluster 2024-07-15T06:39:22.930951+0000 mon.b (mon.1) 2 : cluster [INF] mon.b calling monitor election 2024-07-15T06:39:28.380 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:28 smithi089 bash[31372]: audit 2024-07-15T06:39:22.949152+0000 mon.a (mon.0) 167 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-15T06:39:28.380 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:28 smithi089 bash[31372]: audit 2024-07-15T06:39:23.921644+0000 mon.a (mon.0) 168 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-15T06:39:28.380 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:28 smithi089 bash[31372]: cluster 2024-07-15T06:39:24.867894+0000 mgr.y (mgr.14150) 55 : cluster [DBG] pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:28.380 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:28 smithi089 bash[31372]: audit 2024-07-15T06:39:24.921910+0000 mon.a (mon.0) 169 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-15T06:39:28.381 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:28 smithi089 bash[31372]: cluster 2024-07-15T06:39:24.923960+0000 mon.c (mon.2) 1 : cluster [INF] mon.c calling monitor election 2024-07-15T06:39:28.381 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:28 smithi089 bash[31372]: audit 2024-07-15T06:39:25.921719+0000 mon.a (mon.0) 170 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-15T06:39:28.381 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:28 smithi089 bash[31372]: cluster 2024-07-15T06:39:26.868289+0000 mgr.y (mgr.14150) 56 : cluster [DBG] pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:28.381 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:28 smithi089 bash[31372]: audit 2024-07-15T06:39:26.922033+0000 mon.a (mon.0) 171 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-15T06:39:28.381 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:28 smithi089 bash[31372]: audit 2024-07-15T06:39:27.922173+0000 mon.a (mon.0) 172 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-15T06:39:28.381 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:28 smithi089 bash[31372]: cluster 2024-07-15T06:39:27.933852+0000 mon.a (mon.0) 173 : cluster [INF] mon.a calling monitor election 2024-07-15T06:39:28.381 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:28 smithi089 bash[31372]: cluster 2024-07-15T06:39:27.975155+0000 mon.a (mon.0) 174 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-07-15T06:39:28.381 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:28 smithi089 bash[31372]: cluster 2024-07-15T06:39:27.983988+0000 mon.a (mon.0) 175 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.89:3300/0,v1:172.21.15.89:6789/0],b=[v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0],c=[v2:172.21.15.89:3301/0,v1:172.21.15.89:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-07-15T06:39:28.381 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:28 smithi089 bash[31372]: cluster 2024-07-15T06:39:27.985027+0000 mon.a (mon.0) 176 : cluster [DBG] fsmap 2024-07-15T06:39:28.381 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:28 smithi089 bash[31372]: cluster 2024-07-15T06:39:27.985089+0000 mon.a (mon.0) 177 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2024-07-15T06:39:28.381 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:28 smithi089 bash[31372]: cluster 2024-07-15T06:39:27.985348+0000 mon.a (mon.0) 178 : cluster [DBG] mgrmap e13: y(active, since 81s) 2024-07-15T06:39:28.381 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:28 smithi089 bash[31372]: cluster 2024-07-15T06:39:27.985594+0000 mon.a (mon.0) 179 : cluster [INF] overall HEALTH_OK 2024-07-15T06:39:28.381 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:28 smithi089 bash[31372]: audit 2024-07-15T06:39:27.995090+0000 mon.a (mon.0) 180 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:28.381 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:28 smithi089 bash[31372]: audit 2024-07-15T06:39:28.000875+0000 mon.a (mon.0) 181 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:28.382 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:28 smithi089 bash[31372]: audit 2024-07-15T06:39:28.006581+0000 mon.a (mon.0) 182 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:28.382 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:28 smithi089 bash[31372]: audit 2024-07-15T06:39:28.012287+0000 mon.a (mon.0) 183 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:28.382 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:28 smithi089 bash[31372]: audit 2024-07-15T06:39:28.018016+0000 mon.a (mon.0) 184 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:28.958 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-07-15T06:39:28.958 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph config generate-minimal-conf 2024-07-15T06:39:29.310 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:29 smithi134 bash[24068]: audit 2024-07-15T06:39:28.274940+0000 mon.a (mon.0) 185 : audit [DBG] from='client.? 172.21.15.134:0/294623489' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-07-15T06:39:29.310 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:29 smithi134 bash[24068]: audit 2024-07-15T06:39:28.416293+0000 mon.a (mon.0) 186 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:39:29.310 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:29 smithi134 bash[24068]: audit 2024-07-15T06:39:28.417954+0000 mon.a (mon.0) 187 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:39:29.310 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:29 smithi134 bash[24068]: cephadm 2024-07-15T06:39:28.420044+0000 mgr.y (mgr.14150) 57 : cephadm [INF] Updating smithi089:/etc/ceph/ceph.conf 2024-07-15T06:39:29.310 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:29 smithi134 bash[24068]: cephadm 2024-07-15T06:39:28.420604+0000 mgr.y (mgr.14150) 58 : cephadm [INF] Updating smithi134:/etc/ceph/ceph.conf 2024-07-15T06:39:29.311 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:29 smithi134 bash[24068]: audit 2024-07-15T06:39:28.675905+0000 mon.a (mon.0) 188 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:29.311 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:29 smithi134 bash[24068]: audit 2024-07-15T06:39:28.682733+0000 mon.a (mon.0) 189 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:29.311 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:29 smithi134 bash[24068]: audit 2024-07-15T06:39:28.687749+0000 mon.a (mon.0) 190 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:29.311 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:29 smithi134 bash[24068]: audit 2024-07-15T06:39:28.693472+0000 mon.a (mon.0) 191 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:29.311 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:29 smithi134 bash[24068]: audit 2024-07-15T06:39:28.699650+0000 mon.a (mon.0) 192 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:29.311 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:29 smithi134 bash[24068]: audit 2024-07-15T06:39:28.730538+0000 mon.a (mon.0) 193 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-07-15T06:39:29.311 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:29 smithi134 bash[24068]: audit 2024-07-15T06:39:28.731547+0000 mon.a (mon.0) 194 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-07-15T06:39:29.311 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:29 smithi134 bash[24068]: audit 2024-07-15T06:39:28.732499+0000 mon.a (mon.0) 195 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:39:29.311 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:29 smithi134 bash[24068]: audit 2024-07-15T06:39:28.921497+0000 mon.a (mon.0) 196 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-15T06:39:29.375 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:29 smithi089 bash[21793]: audit 2024-07-15T06:39:28.274940+0000 mon.a (mon.0) 185 : audit [DBG] from='client.? 172.21.15.134:0/294623489' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-07-15T06:39:29.375 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:29 smithi089 bash[21793]: audit 2024-07-15T06:39:28.416293+0000 mon.a (mon.0) 186 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:39:29.375 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:29 smithi089 bash[21793]: audit 2024-07-15T06:39:28.417954+0000 mon.a (mon.0) 187 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:39:29.375 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:29 smithi089 bash[21793]: cephadm 2024-07-15T06:39:28.420044+0000 mgr.y (mgr.14150) 57 : cephadm [INF] Updating smithi089:/etc/ceph/ceph.conf 2024-07-15T06:39:29.375 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:29 smithi089 bash[21793]: cephadm 2024-07-15T06:39:28.420604+0000 mgr.y (mgr.14150) 58 : cephadm [INF] Updating smithi134:/etc/ceph/ceph.conf 2024-07-15T06:39:29.376 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:29 smithi089 bash[21793]: audit 2024-07-15T06:39:28.675905+0000 mon.a (mon.0) 188 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:29.376 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:29 smithi089 bash[21793]: audit 2024-07-15T06:39:28.682733+0000 mon.a (mon.0) 189 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:29.376 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:29 smithi089 bash[21793]: audit 2024-07-15T06:39:28.687749+0000 mon.a (mon.0) 190 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:29.376 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:29 smithi089 bash[21793]: audit 2024-07-15T06:39:28.693472+0000 mon.a (mon.0) 191 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:29.376 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:29 smithi089 bash[21793]: audit 2024-07-15T06:39:28.699650+0000 mon.a (mon.0) 192 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:29.376 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:29 smithi089 bash[21793]: audit 2024-07-15T06:39:28.730538+0000 mon.a (mon.0) 193 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-07-15T06:39:29.376 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:29 smithi089 bash[21793]: audit 2024-07-15T06:39:28.731547+0000 mon.a (mon.0) 194 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-07-15T06:39:29.376 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:29 smithi089 bash[21793]: audit 2024-07-15T06:39:28.732499+0000 mon.a (mon.0) 195 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:39:29.376 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:29 smithi089 bash[21793]: audit 2024-07-15T06:39:28.921497+0000 mon.a (mon.0) 196 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-15T06:39:29.376 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:29 smithi089 bash[31372]: audit 2024-07-15T06:39:28.274940+0000 mon.a (mon.0) 185 : audit [DBG] from='client.? 172.21.15.134:0/294623489' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-07-15T06:39:29.376 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:29 smithi089 bash[31372]: audit 2024-07-15T06:39:28.416293+0000 mon.a (mon.0) 186 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:39:29.376 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:29 smithi089 bash[31372]: audit 2024-07-15T06:39:28.417954+0000 mon.a (mon.0) 187 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:39:29.377 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:29 smithi089 bash[31372]: cephadm 2024-07-15T06:39:28.420044+0000 mgr.y (mgr.14150) 57 : cephadm [INF] Updating smithi089:/etc/ceph/ceph.conf 2024-07-15T06:39:29.377 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:29 smithi089 bash[31372]: cephadm 2024-07-15T06:39:28.420604+0000 mgr.y (mgr.14150) 58 : cephadm [INF] Updating smithi134:/etc/ceph/ceph.conf 2024-07-15T06:39:29.377 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:29 smithi089 bash[31372]: audit 2024-07-15T06:39:28.675905+0000 mon.a (mon.0) 188 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:29.377 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:29 smithi089 bash[31372]: audit 2024-07-15T06:39:28.682733+0000 mon.a (mon.0) 189 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:29.377 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:29 smithi089 bash[31372]: audit 2024-07-15T06:39:28.687749+0000 mon.a (mon.0) 190 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:29.377 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:29 smithi089 bash[31372]: audit 2024-07-15T06:39:28.693472+0000 mon.a (mon.0) 191 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:29.377 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:29 smithi089 bash[31372]: audit 2024-07-15T06:39:28.699650+0000 mon.a (mon.0) 192 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:29.377 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:29 smithi089 bash[31372]: audit 2024-07-15T06:39:28.730538+0000 mon.a (mon.0) 193 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-07-15T06:39:29.377 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:29 smithi089 bash[31372]: audit 2024-07-15T06:39:28.731547+0000 mon.a (mon.0) 194 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-07-15T06:39:29.377 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:29 smithi089 bash[31372]: audit 2024-07-15T06:39:28.732499+0000 mon.a (mon.0) 195 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:39:29.377 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:29 smithi089 bash[31372]: audit 2024-07-15T06:39:28.921497+0000 mon.a (mon.0) 196 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-15T06:39:30.310 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:30 smithi134 bash[24068]: cephadm 2024-07-15T06:39:28.554024+0000 mgr.y (mgr.14150) 59 : cephadm [INF] Updating smithi089:/var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/config/ceph.conf 2024-07-15T06:39:30.310 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:30 smithi134 bash[24068]: cephadm 2024-07-15T06:39:28.557794+0000 mgr.y (mgr.14150) 60 : cephadm [INF] Updating smithi134:/var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/config/ceph.conf 2024-07-15T06:39:30.311 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:30 smithi134 bash[24068]: cephadm 2024-07-15T06:39:28.730100+0000 mgr.y (mgr.14150) 61 : cephadm [INF] Reconfiguring mon.a (unknown last config time)... 2024-07-15T06:39:30.311 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:30 smithi134 bash[24068]: cephadm 2024-07-15T06:39:28.733574+0000 mgr.y (mgr.14150) 62 : cephadm [INF] Reconfiguring daemon mon.a on smithi089 2024-07-15T06:39:30.311 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:30 smithi134 bash[24068]: cluster 2024-07-15T06:39:28.868686+0000 mgr.y (mgr.14150) 63 : cluster [DBG] pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:30.351 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:30 smithi089 bash[31372]: cephadm 2024-07-15T06:39:28.554024+0000 mgr.y (mgr.14150) 59 : cephadm [INF] Updating smithi089:/var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/config/ceph.conf 2024-07-15T06:39:30.351 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:30 smithi089 bash[31372]: cephadm 2024-07-15T06:39:28.557794+0000 mgr.y (mgr.14150) 60 : cephadm [INF] Updating smithi134:/var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/config/ceph.conf 2024-07-15T06:39:30.351 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:30 smithi089 bash[31372]: cephadm 2024-07-15T06:39:28.730100+0000 mgr.y (mgr.14150) 61 : cephadm [INF] Reconfiguring mon.a (unknown last config time)... 2024-07-15T06:39:30.351 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:30 smithi089 bash[31372]: cephadm 2024-07-15T06:39:28.733574+0000 mgr.y (mgr.14150) 62 : cephadm [INF] Reconfiguring daemon mon.a on smithi089 2024-07-15T06:39:30.351 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:30 smithi089 bash[31372]: cluster 2024-07-15T06:39:28.868686+0000 mgr.y (mgr.14150) 63 : cluster [DBG] pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:30.352 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:30 smithi089 bash[21793]: cephadm 2024-07-15T06:39:28.554024+0000 mgr.y (mgr.14150) 59 : cephadm [INF] Updating smithi089:/var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/config/ceph.conf 2024-07-15T06:39:30.352 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:30 smithi089 bash[21793]: cephadm 2024-07-15T06:39:28.557794+0000 mgr.y (mgr.14150) 60 : cephadm [INF] Updating smithi134:/var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/config/ceph.conf 2024-07-15T06:39:30.352 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:30 smithi089 bash[21793]: cephadm 2024-07-15T06:39:28.730100+0000 mgr.y (mgr.14150) 61 : cephadm [INF] Reconfiguring mon.a (unknown last config time)... 2024-07-15T06:39:30.352 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:30 smithi089 bash[21793]: cephadm 2024-07-15T06:39:28.733574+0000 mgr.y (mgr.14150) 62 : cephadm [INF] Reconfiguring daemon mon.a on smithi089 2024-07-15T06:39:30.352 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:30 smithi089 bash[21793]: cluster 2024-07-15T06:39:28.868686+0000 mgr.y (mgr.14150) 63 : cluster [DBG] pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:32.211 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:31 smithi089 bash[31372]: cluster 2024-07-15T06:39:30.869028+0000 mgr.y (mgr.14150) 64 : cluster [DBG] pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:32.211 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:31 smithi089 bash[31372]: audit 2024-07-15T06:39:30.900435+0000 mon.a (mon.0) 197 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:32.211 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:31 smithi089 bash[31372]: audit 2024-07-15T06:39:30.905027+0000 mon.a (mon.0) 198 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:32.212 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:31 smithi089 bash[31372]: cephadm 2024-07-15T06:39:30.905896+0000 mgr.y (mgr.14150) 65 : cephadm [INF] Reconfiguring mon.c (monmap changed)... 2024-07-15T06:39:32.212 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:31 smithi089 bash[31372]: audit 2024-07-15T06:39:30.906206+0000 mon.a (mon.0) 199 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-07-15T06:39:32.212 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:31 smithi089 bash[31372]: audit 2024-07-15T06:39:30.907170+0000 mon.a (mon.0) 200 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-07-15T06:39:32.212 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:31 smithi089 bash[31372]: audit 2024-07-15T06:39:30.908114+0000 mon.a (mon.0) 201 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:39:32.212 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:31 smithi089 bash[31372]: cephadm 2024-07-15T06:39:30.909179+0000 mgr.y (mgr.14150) 66 : cephadm [INF] Reconfiguring daemon mon.c on smithi089 2024-07-15T06:39:32.212 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:31 smithi089 bash[21793]: cluster 2024-07-15T06:39:30.869028+0000 mgr.y (mgr.14150) 64 : cluster [DBG] pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:32.212 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:31 smithi089 bash[21793]: audit 2024-07-15T06:39:30.900435+0000 mon.a (mon.0) 197 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:32.212 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:31 smithi089 bash[21793]: audit 2024-07-15T06:39:30.905027+0000 mon.a (mon.0) 198 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:32.212 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:31 smithi089 bash[21793]: cephadm 2024-07-15T06:39:30.905896+0000 mgr.y (mgr.14150) 65 : cephadm [INF] Reconfiguring mon.c (monmap changed)... 2024-07-15T06:39:32.212 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:31 smithi089 bash[21793]: audit 2024-07-15T06:39:30.906206+0000 mon.a (mon.0) 199 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-07-15T06:39:32.212 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:31 smithi089 bash[21793]: audit 2024-07-15T06:39:30.907170+0000 mon.a (mon.0) 200 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-07-15T06:39:32.213 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:31 smithi089 bash[21793]: audit 2024-07-15T06:39:30.908114+0000 mon.a (mon.0) 201 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:39:32.213 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:31 smithi089 bash[21793]: cephadm 2024-07-15T06:39:30.909179+0000 mgr.y (mgr.14150) 66 : cephadm [INF] Reconfiguring daemon mon.c on smithi089 2024-07-15T06:39:32.310 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:31 smithi134 bash[24068]: cluster 2024-07-15T06:39:30.869028+0000 mgr.y (mgr.14150) 64 : cluster [DBG] pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:32.310 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:31 smithi134 bash[24068]: audit 2024-07-15T06:39:30.900435+0000 mon.a (mon.0) 197 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:32.310 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:31 smithi134 bash[24068]: audit 2024-07-15T06:39:30.905027+0000 mon.a (mon.0) 198 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:32.310 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:31 smithi134 bash[24068]: cephadm 2024-07-15T06:39:30.905896+0000 mgr.y (mgr.14150) 65 : cephadm [INF] Reconfiguring mon.c (monmap changed)... 2024-07-15T06:39:32.310 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:31 smithi134 bash[24068]: audit 2024-07-15T06:39:30.906206+0000 mon.a (mon.0) 199 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-07-15T06:39:32.310 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:31 smithi134 bash[24068]: audit 2024-07-15T06:39:30.907170+0000 mon.a (mon.0) 200 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-07-15T06:39:32.310 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:31 smithi134 bash[24068]: audit 2024-07-15T06:39:30.908114+0000 mon.a (mon.0) 201 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:39:32.311 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:31 smithi134 bash[24068]: cephadm 2024-07-15T06:39:30.909179+0000 mgr.y (mgr.14150) 66 : cephadm [INF] Reconfiguring daemon mon.c on smithi089 2024-07-15T06:39:33.674 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T06:39:34.059 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:33 smithi134 bash[24068]: audit 2024-07-15T06:39:32.797283+0000 mon.a (mon.0) 202 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:34.060 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:33 smithi134 bash[24068]: audit 2024-07-15T06:39:32.803935+0000 mon.a (mon.0) 203 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:34.060 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:33 smithi134 bash[24068]: cephadm 2024-07-15T06:39:32.804733+0000 mgr.y (mgr.14150) 67 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2024-07-15T06:39:34.060 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:33 smithi134 bash[24068]: audit 2024-07-15T06:39:32.805113+0000 mon.a (mon.0) 204 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-07-15T06:39:34.060 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:33 smithi134 bash[24068]: audit 2024-07-15T06:39:32.806281+0000 mon.a (mon.0) 205 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-07-15T06:39:34.060 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:33 smithi134 bash[24068]: audit 2024-07-15T06:39:32.807096+0000 mon.a (mon.0) 206 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:39:34.060 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:33 smithi134 bash[24068]: cephadm 2024-07-15T06:39:32.808121+0000 mgr.y (mgr.14150) 68 : cephadm [INF] Reconfiguring daemon mon.b on smithi134 2024-07-15T06:39:34.060 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:33 smithi134 bash[24068]: cluster 2024-07-15T06:39:32.869479+0000 mgr.y (mgr.14150) 69 : cluster [DBG] pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:34.125 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:33 smithi089 bash[21793]: audit 2024-07-15T06:39:32.797283+0000 mon.a (mon.0) 202 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:34.125 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:33 smithi089 bash[21793]: audit 2024-07-15T06:39:32.803935+0000 mon.a (mon.0) 203 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:34.125 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:33 smithi089 bash[21793]: cephadm 2024-07-15T06:39:32.804733+0000 mgr.y (mgr.14150) 67 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2024-07-15T06:39:34.125 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:33 smithi089 bash[21793]: audit 2024-07-15T06:39:32.805113+0000 mon.a (mon.0) 204 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-07-15T06:39:34.125 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:33 smithi089 bash[21793]: audit 2024-07-15T06:39:32.806281+0000 mon.a (mon.0) 205 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-07-15T06:39:34.125 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:33 smithi089 bash[21793]: audit 2024-07-15T06:39:32.807096+0000 mon.a (mon.0) 206 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:39:34.125 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:33 smithi089 bash[21793]: cephadm 2024-07-15T06:39:32.808121+0000 mgr.y (mgr.14150) 68 : cephadm [INF] Reconfiguring daemon mon.b on smithi134 2024-07-15T06:39:34.126 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:33 smithi089 bash[21793]: cluster 2024-07-15T06:39:32.869479+0000 mgr.y (mgr.14150) 69 : cluster [DBG] pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:34.126 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:33 smithi089 bash[31372]: audit 2024-07-15T06:39:32.797283+0000 mon.a (mon.0) 202 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:34.126 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:33 smithi089 bash[31372]: audit 2024-07-15T06:39:32.803935+0000 mon.a (mon.0) 203 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:34.126 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:33 smithi089 bash[31372]: cephadm 2024-07-15T06:39:32.804733+0000 mgr.y (mgr.14150) 67 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2024-07-15T06:39:34.126 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:33 smithi089 bash[31372]: audit 2024-07-15T06:39:32.805113+0000 mon.a (mon.0) 204 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-07-15T06:39:34.126 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:33 smithi089 bash[31372]: audit 2024-07-15T06:39:32.806281+0000 mon.a (mon.0) 205 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-07-15T06:39:34.126 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:33 smithi089 bash[31372]: audit 2024-07-15T06:39:32.807096+0000 mon.a (mon.0) 206 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:39:34.126 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:33 smithi089 bash[31372]: cephadm 2024-07-15T06:39:32.808121+0000 mgr.y (mgr.14150) 68 : cephadm [INF] Reconfiguring daemon mon.b on smithi134 2024-07-15T06:39:34.126 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:33 smithi089 bash[31372]: cluster 2024-07-15T06:39:32.869479+0000 mgr.y (mgr.14150) 69 : cluster [DBG] pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:36.060 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:35 smithi134 bash[24068]: audit 2024-07-15T06:39:34.785335+0000 mon.a (mon.0) 207 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:36.060 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:35 smithi134 bash[24068]: audit 2024-07-15T06:39:34.794539+0000 mon.a (mon.0) 208 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:36.060 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:35 smithi134 bash[24068]: audit 2024-07-15T06:39:34.796664+0000 mon.a (mon.0) 209 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:39:36.060 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:35 smithi134 bash[24068]: audit 2024-07-15T06:39:34.799912+0000 mon.a (mon.0) 210 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:39:36.060 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:35 smithi134 bash[24068]: audit 2024-07-15T06:39:34.801631+0000 mon.a (mon.0) 211 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:39:36.060 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:35 smithi134 bash[24068]: audit 2024-07-15T06:39:34.811331+0000 mon.a (mon.0) 212 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:36.060 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:35 smithi134 bash[24068]: cluster 2024-07-15T06:39:34.869759+0000 mgr.y (mgr.14150) 70 : cluster [DBG] pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:36.125 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:35 smithi089 bash[31372]: audit 2024-07-15T06:39:34.785335+0000 mon.a (mon.0) 207 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:36.125 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:35 smithi089 bash[31372]: audit 2024-07-15T06:39:34.794539+0000 mon.a (mon.0) 208 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:36.125 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:35 smithi089 bash[31372]: audit 2024-07-15T06:39:34.796664+0000 mon.a (mon.0) 209 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:39:36.125 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:35 smithi089 bash[31372]: audit 2024-07-15T06:39:34.799912+0000 mon.a (mon.0) 210 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:39:36.125 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:35 smithi089 bash[31372]: audit 2024-07-15T06:39:34.801631+0000 mon.a (mon.0) 211 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:39:36.125 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:35 smithi089 bash[31372]: audit 2024-07-15T06:39:34.811331+0000 mon.a (mon.0) 212 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:36.125 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:35 smithi089 bash[31372]: cluster 2024-07-15T06:39:34.869759+0000 mgr.y (mgr.14150) 70 : cluster [DBG] pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:36.125 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:35 smithi089 bash[21793]: audit 2024-07-15T06:39:34.785335+0000 mon.a (mon.0) 207 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:36.126 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:35 smithi089 bash[21793]: audit 2024-07-15T06:39:34.794539+0000 mon.a (mon.0) 208 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:36.126 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:35 smithi089 bash[21793]: audit 2024-07-15T06:39:34.796664+0000 mon.a (mon.0) 209 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:39:36.126 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:35 smithi089 bash[21793]: audit 2024-07-15T06:39:34.799912+0000 mon.a (mon.0) 210 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:39:36.126 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:35 smithi089 bash[21793]: audit 2024-07-15T06:39:34.801631+0000 mon.a (mon.0) 211 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:39:36.126 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:35 smithi089 bash[21793]: audit 2024-07-15T06:39:34.811331+0000 mon.a (mon.0) 212 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:36.126 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:35 smithi089 bash[21793]: cluster 2024-07-15T06:39:34.869759+0000 mgr.y (mgr.14150) 70 : cluster [DBG] pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:36.992 INFO:teuthology.orchestra.run.smithi089.stdout:# minimal ceph.conf for 75ba02bc-4274-11ef-bcad-c7b262605968 2024-07-15T06:39:36.993 INFO:teuthology.orchestra.run.smithi089.stdout:[global] 2024-07-15T06:39:36.993 INFO:teuthology.orchestra.run.smithi089.stdout: fsid = 75ba02bc-4274-11ef-bcad-c7b262605968 2024-07-15T06:39:36.993 INFO:teuthology.orchestra.run.smithi089.stdout: mon_host = [v2:172.21.15.89:3300/0,v1:172.21.15.89:6789/0] [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] [v2:172.21.15.89:3301/0,v1:172.21.15.89:6790/0] 2024-07-15T06:39:37.582 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-07-15T06:39:37.582 DEBUG:teuthology.orchestra.run.smithi089:> set -ex 2024-07-15T06:39:37.582 DEBUG:teuthology.orchestra.run.smithi089:> sudo dd of=/etc/ceph/ceph.conf 2024-07-15T06:39:37.597 DEBUG:teuthology.orchestra.run.smithi089:> set -ex 2024-07-15T06:39:37.597 DEBUG:teuthology.orchestra.run.smithi089:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-07-15T06:39:37.651 DEBUG:teuthology.orchestra.run.smithi134:> set -ex 2024-07-15T06:39:37.651 DEBUG:teuthology.orchestra.run.smithi134:> sudo dd of=/etc/ceph/ceph.conf 2024-07-15T06:39:37.666 DEBUG:teuthology.orchestra.run.smithi134:> set -ex 2024-07-15T06:39:37.666 DEBUG:teuthology.orchestra.run.smithi134:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-07-15T06:39:37.721 INFO:tasks.cephadm:Adding mgr.y on smithi089 2024-07-15T06:39:37.721 INFO:tasks.cephadm:Adding mgr.x on smithi134 2024-07-15T06:39:37.722 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph orch apply mgr '2;smithi089=y;smithi134=x' 2024-07-15T06:39:38.310 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:37 smithi134 bash[24068]: cluster 2024-07-15T06:39:36.869992+0000 mgr.y (mgr.14150) 71 : cluster [DBG] pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:38.310 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:37 smithi134 bash[24068]: audit 2024-07-15T06:39:36.992690+0000 mon.a (mon.0) 213 : audit [DBG] from='client.? 172.21.15.89:0/3061533463' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:39:38.375 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:37 smithi089 bash[21793]: cluster 2024-07-15T06:39:36.869992+0000 mgr.y (mgr.14150) 71 : cluster [DBG] pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:38.375 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:37 smithi089 bash[21793]: audit 2024-07-15T06:39:36.992690+0000 mon.a (mon.0) 213 : audit [DBG] from='client.? 172.21.15.89:0/3061533463' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:39:38.375 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:37 smithi089 bash[31372]: cluster 2024-07-15T06:39:36.869992+0000 mgr.y (mgr.14150) 71 : cluster [DBG] pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:38.375 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:37 smithi089 bash[31372]: audit 2024-07-15T06:39:36.992690+0000 mon.a (mon.0) 213 : audit [DBG] from='client.? 172.21.15.89:0/3061533463' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:39:40.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:39 smithi134 bash[24068]: cluster 2024-07-15T06:39:38.870376+0000 mgr.y (mgr.14150) 72 : cluster [DBG] pgmap v40: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:40.374 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:39 smithi089 bash[21793]: cluster 2024-07-15T06:39:38.870376+0000 mgr.y (mgr.14150) 72 : cluster [DBG] pgmap v40: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:40.375 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:39 smithi089 bash[31372]: cluster 2024-07-15T06:39:38.870376+0000 mgr.y (mgr.14150) 72 : cluster [DBG] pgmap v40: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:41.462 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.b/config 2024-07-15T06:39:42.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:41 smithi134 bash[24068]: cluster 2024-07-15T06:39:40.870780+0000 mgr.y (mgr.14150) 73 : cluster [DBG] pgmap v41: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:42.374 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:41 smithi089 bash[21793]: cluster 2024-07-15T06:39:40.870780+0000 mgr.y (mgr.14150) 73 : cluster [DBG] pgmap v41: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:42.375 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:41 smithi089 bash[31372]: cluster 2024-07-15T06:39:40.870780+0000 mgr.y (mgr.14150) 73 : cluster [DBG] pgmap v41: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:44.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:43 smithi134 bash[24068]: cluster 2024-07-15T06:39:42.871185+0000 mgr.y (mgr.14150) 74 : cluster [DBG] pgmap v42: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:44.375 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:43 smithi089 bash[21793]: cluster 2024-07-15T06:39:42.871185+0000 mgr.y (mgr.14150) 74 : cluster [DBG] pgmap v42: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:44.375 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:43 smithi089 bash[31372]: cluster 2024-07-15T06:39:42.871185+0000 mgr.y (mgr.14150) 74 : cluster [DBG] pgmap v42: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:44.451 INFO:teuthology.orchestra.run.smithi134.stdout:Scheduled mgr update... 2024-07-15T06:39:45.310 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:44 smithi134 bash[24068]: audit 2024-07-15T06:39:44.441053+0000 mgr.y (mgr.14150) 75 : audit [DBG] from='client.24103 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi089=y;smithi134=x", "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T06:39:45.310 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:44 smithi134 bash[24068]: audit 2024-07-15T06:39:44.450419+0000 mon.a (mon.0) 214 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:45.310 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:44 smithi134 bash[24068]: audit 2024-07-15T06:39:44.452114+0000 mon.a (mon.0) 215 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:39:45.310 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:44 smithi134 bash[24068]: audit 2024-07-15T06:39:44.455487+0000 mon.a (mon.0) 216 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:39:45.310 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:44 smithi134 bash[24068]: audit 2024-07-15T06:39:44.457193+0000 mon.a (mon.0) 217 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:39:45.310 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:44 smithi134 bash[24068]: audit 2024-07-15T06:39:44.465934+0000 mon.a (mon.0) 218 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:45.310 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:44 smithi134 bash[24068]: audit 2024-07-15T06:39:44.469175+0000 mon.a (mon.0) 219 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-07-15T06:39:45.311 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:44 smithi134 bash[24068]: audit 2024-07-15T06:39:44.474986+0000 mon.a (mon.0) 220 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-07-15T06:39:45.311 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:44 smithi134 bash[24068]: audit 2024-07-15T06:39:44.476771+0000 mon.a (mon.0) 221 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-07-15T06:39:45.311 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:44 smithi134 bash[24068]: audit 2024-07-15T06:39:44.478733+0000 mon.a (mon.0) 222 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:39:45.375 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:44 smithi089 bash[21793]: audit 2024-07-15T06:39:44.441053+0000 mgr.y (mgr.14150) 75 : audit [DBG] from='client.24103 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi089=y;smithi134=x", "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T06:39:45.375 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:44 smithi089 bash[21793]: audit 2024-07-15T06:39:44.450419+0000 mon.a (mon.0) 214 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:45.375 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:44 smithi089 bash[21793]: audit 2024-07-15T06:39:44.452114+0000 mon.a (mon.0) 215 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:39:45.375 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:44 smithi089 bash[21793]: audit 2024-07-15T06:39:44.455487+0000 mon.a (mon.0) 216 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:39:45.375 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:44 smithi089 bash[21793]: audit 2024-07-15T06:39:44.457193+0000 mon.a (mon.0) 217 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:39:45.375 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:44 smithi089 bash[21793]: audit 2024-07-15T06:39:44.465934+0000 mon.a (mon.0) 218 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:45.376 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:44 smithi089 bash[21793]: audit 2024-07-15T06:39:44.469175+0000 mon.a (mon.0) 219 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-07-15T06:39:45.376 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:44 smithi089 bash[21793]: audit 2024-07-15T06:39:44.474986+0000 mon.a (mon.0) 220 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-07-15T06:39:45.376 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:44 smithi089 bash[21793]: audit 2024-07-15T06:39:44.476771+0000 mon.a (mon.0) 221 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-07-15T06:39:45.376 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:44 smithi089 bash[21793]: audit 2024-07-15T06:39:44.478733+0000 mon.a (mon.0) 222 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:39:45.376 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:44 smithi089 bash[31372]: audit 2024-07-15T06:39:44.441053+0000 mgr.y (mgr.14150) 75 : audit [DBG] from='client.24103 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi089=y;smithi134=x", "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T06:39:45.376 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:44 smithi089 bash[31372]: audit 2024-07-15T06:39:44.450419+0000 mon.a (mon.0) 214 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:45.376 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:44 smithi089 bash[31372]: audit 2024-07-15T06:39:44.452114+0000 mon.a (mon.0) 215 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:39:45.376 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:44 smithi089 bash[31372]: audit 2024-07-15T06:39:44.455487+0000 mon.a (mon.0) 216 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:39:45.376 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:44 smithi089 bash[31372]: audit 2024-07-15T06:39:44.457193+0000 mon.a (mon.0) 217 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:39:45.377 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:44 smithi089 bash[31372]: audit 2024-07-15T06:39:44.465934+0000 mon.a (mon.0) 218 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:45.377 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:44 smithi089 bash[31372]: audit 2024-07-15T06:39:44.469175+0000 mon.a (mon.0) 219 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-07-15T06:39:45.377 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:44 smithi089 bash[31372]: audit 2024-07-15T06:39:44.474986+0000 mon.a (mon.0) 220 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-07-15T06:39:45.377 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:44 smithi089 bash[31372]: audit 2024-07-15T06:39:44.476771+0000 mon.a (mon.0) 221 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-07-15T06:39:45.377 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:44 smithi089 bash[31372]: audit 2024-07-15T06:39:44.478733+0000 mon.a (mon.0) 222 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:39:45.415 DEBUG:teuthology.orchestra.run.smithi134:mgr.x> sudo journalctl -f -n 0 -u ceph-75ba02bc-4274-11ef-bcad-c7b262605968@mgr.x.service 2024-07-15T06:39:45.417 INFO:tasks.cephadm:Deploying OSDs... 2024-07-15T06:39:45.417 DEBUG:teuthology.orchestra.run.smithi089:> set -ex 2024-07-15T06:39:45.417 DEBUG:teuthology.orchestra.run.smithi089:> dd if=/scratch_devs of=/dev/stdout 2024-07-15T06:39:45.424 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-07-15T06:39:45.424 DEBUG:teuthology.orchestra.run.smithi089:> stat /dev/vg_nvme/lv_1 2024-07-15T06:39:45.472 INFO:teuthology.orchestra.run.smithi089.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-07-15T06:39:45.472 INFO:teuthology.orchestra.run.smithi089.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-07-15T06:39:45.472 INFO:teuthology.orchestra.run.smithi089.stdout:Device: 5h/5d Inode: 870 Links: 1 2024-07-15T06:39:45.472 INFO:teuthology.orchestra.run.smithi089.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-07-15T06:39:45.472 INFO:teuthology.orchestra.run.smithi089.stdout:Access: 2024-07-15 06:30:41.679671390 +0000 2024-07-15T06:39:45.472 INFO:teuthology.orchestra.run.smithi089.stdout:Modify: 2024-07-15 06:30:41.675671424 +0000 2024-07-15T06:39:45.472 INFO:teuthology.orchestra.run.smithi089.stdout:Change: 2024-07-15 06:30:41.675671424 +0000 2024-07-15T06:39:45.472 INFO:teuthology.orchestra.run.smithi089.stdout: Birth: - 2024-07-15T06:39:45.473 DEBUG:teuthology.orchestra.run.smithi089:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-07-15T06:39:45.526 INFO:teuthology.orchestra.run.smithi089.stderr:1+0 records in 2024-07-15T06:39:45.527 INFO:teuthology.orchestra.run.smithi089.stderr:1+0 records out 2024-07-15T06:39:45.527 INFO:teuthology.orchestra.run.smithi089.stderr:512 bytes copied, 0.000442379 s, 1.2 MB/s 2024-07-15T06:39:45.528 DEBUG:teuthology.orchestra.run.smithi089:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-07-15T06:39:45.576 DEBUG:teuthology.orchestra.run.smithi089:> stat /dev/vg_nvme/lv_2 2024-07-15T06:39:45.623 INFO:teuthology.orchestra.run.smithi089.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-07-15T06:39:45.623 INFO:teuthology.orchestra.run.smithi089.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-07-15T06:39:45.623 INFO:teuthology.orchestra.run.smithi089.stdout:Device: 5h/5d Inode: 888 Links: 1 2024-07-15T06:39:45.623 INFO:teuthology.orchestra.run.smithi089.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-07-15T06:39:45.623 INFO:teuthology.orchestra.run.smithi089.stdout:Access: 2024-07-15 06:30:42.403664923 +0000 2024-07-15T06:39:45.623 INFO:teuthology.orchestra.run.smithi089.stdout:Modify: 2024-07-15 06:30:42.167667031 +0000 2024-07-15T06:39:45.623 INFO:teuthology.orchestra.run.smithi089.stdout:Change: 2024-07-15 06:30:42.167667031 +0000 2024-07-15T06:39:45.624 INFO:teuthology.orchestra.run.smithi089.stdout: Birth: - 2024-07-15T06:39:45.624 DEBUG:teuthology.orchestra.run.smithi089:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-07-15T06:39:45.678 INFO:teuthology.orchestra.run.smithi089.stderr:1+0 records in 2024-07-15T06:39:45.678 INFO:teuthology.orchestra.run.smithi089.stderr:1+0 records out 2024-07-15T06:39:45.678 INFO:teuthology.orchestra.run.smithi089.stderr:512 bytes copied, 0.000452086 s, 1.1 MB/s 2024-07-15T06:39:45.679 DEBUG:teuthology.orchestra.run.smithi089:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-07-15T06:39:45.728 DEBUG:teuthology.orchestra.run.smithi089:> stat /dev/vg_nvme/lv_3 2024-07-15T06:39:45.792 INFO:teuthology.orchestra.run.smithi089.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-07-15T06:39:45.792 INFO:teuthology.orchestra.run.smithi089.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-07-15T06:39:45.792 INFO:teuthology.orchestra.run.smithi089.stdout:Device: 5h/5d Inode: 903 Links: 1 2024-07-15T06:39:45.792 INFO:teuthology.orchestra.run.smithi089.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-07-15T06:39:45.792 INFO:teuthology.orchestra.run.smithi089.stdout:Access: 2024-07-15 06:30:42.871660743 +0000 2024-07-15T06:39:45.793 INFO:teuthology.orchestra.run.smithi089.stdout:Modify: 2024-07-15 06:30:42.639662815 +0000 2024-07-15T06:39:45.793 INFO:teuthology.orchestra.run.smithi089.stdout:Change: 2024-07-15 06:30:42.639662815 +0000 2024-07-15T06:39:45.793 INFO:teuthology.orchestra.run.smithi089.stdout: Birth: - 2024-07-15T06:39:45.793 DEBUG:teuthology.orchestra.run.smithi089:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-07-15T06:39:45.846 INFO:teuthology.orchestra.run.smithi089.stderr:1+0 records in 2024-07-15T06:39:45.846 INFO:teuthology.orchestra.run.smithi089.stderr:1+0 records out 2024-07-15T06:39:45.846 INFO:teuthology.orchestra.run.smithi089.stderr:512 bytes copied, 0.00040017 s, 1.3 MB/s 2024-07-15T06:39:45.847 DEBUG:teuthology.orchestra.run.smithi089:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-07-15T06:39:45.897 DEBUG:teuthology.orchestra.run.smithi089:> stat /dev/vg_nvme/lv_4 2024-07-15T06:39:45.944 INFO:teuthology.orchestra.run.smithi089.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-07-15T06:39:45.944 INFO:teuthology.orchestra.run.smithi089.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-07-15T06:39:45.944 INFO:teuthology.orchestra.run.smithi089.stdout:Device: 5h/5d Inode: 923 Links: 1 2024-07-15T06:39:45.944 INFO:teuthology.orchestra.run.smithi089.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-07-15T06:39:45.944 INFO:teuthology.orchestra.run.smithi089.stdout:Access: 2024-07-15 06:30:43.359656383 +0000 2024-07-15T06:39:45.944 INFO:teuthology.orchestra.run.smithi089.stdout:Modify: 2024-07-15 06:30:43.107658634 +0000 2024-07-15T06:39:45.944 INFO:teuthology.orchestra.run.smithi089.stdout:Change: 2024-07-15 06:30:43.107658634 +0000 2024-07-15T06:39:45.944 INFO:teuthology.orchestra.run.smithi089.stdout: Birth: - 2024-07-15T06:39:45.945 DEBUG:teuthology.orchestra.run.smithi089:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-07-15T06:39:45.998 INFO:teuthology.orchestra.run.smithi089.stderr:1+0 records in 2024-07-15T06:39:45.998 INFO:teuthology.orchestra.run.smithi089.stderr:1+0 records out 2024-07-15T06:39:45.998 INFO:teuthology.orchestra.run.smithi089.stderr:512 bytes copied, 0.000399217 s, 1.3 MB/s 2024-07-15T06:39:46.000 DEBUG:teuthology.orchestra.run.smithi089:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-07-15T06:39:46.049 DEBUG:teuthology.orchestra.run.smithi134:> set -ex 2024-07-15T06:39:46.049 DEBUG:teuthology.orchestra.run.smithi134:> dd if=/scratch_devs of=/dev/stdout 2024-07-15T06:39:46.056 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-07-15T06:39:46.057 DEBUG:teuthology.orchestra.run.smithi134:> stat /dev/vg_nvme/lv_1 2024-07-15T06:39:46.105 INFO:teuthology.orchestra.run.smithi134.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-07-15T06:39:46.105 INFO:teuthology.orchestra.run.smithi134.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-07-15T06:39:46.106 INFO:teuthology.orchestra.run.smithi134.stdout:Device: 5h/5d Inode: 870 Links: 1 2024-07-15T06:39:46.106 INFO:teuthology.orchestra.run.smithi134.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-07-15T06:39:46.106 INFO:teuthology.orchestra.run.smithi134.stdout:Access: 2024-07-15 06:30:44.407680642 +0000 2024-07-15T06:39:46.106 INFO:teuthology.orchestra.run.smithi134.stdout:Modify: 2024-07-15 06:30:44.167682784 +0000 2024-07-15T06:39:46.106 INFO:teuthology.orchestra.run.smithi134.stdout:Change: 2024-07-15 06:30:44.167682784 +0000 2024-07-15T06:39:46.106 INFO:teuthology.orchestra.run.smithi134.stdout: Birth: - 2024-07-15T06:39:46.106 DEBUG:teuthology.orchestra.run.smithi134:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-07-15T06:39:46.125 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:45 smithi089 bash[31372]: cephadm 2024-07-15T06:39:44.444017+0000 mgr.y (mgr.14150) 76 : cephadm [INF] Saving service mgr spec with placement smithi089=y;smithi134=x;count:2 2024-07-15T06:39:46.125 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:45 smithi089 bash[31372]: cephadm 2024-07-15T06:39:44.480741+0000 mgr.y (mgr.14150) 77 : cephadm [INF] Deploying daemon mgr.x on smithi134 2024-07-15T06:39:46.125 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:45 smithi089 bash[31372]: cluster 2024-07-15T06:39:44.871571+0000 mgr.y (mgr.14150) 78 : cluster [DBG] pgmap v43: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:46.125 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:45 smithi089 bash[21793]: cephadm 2024-07-15T06:39:44.444017+0000 mgr.y (mgr.14150) 76 : cephadm [INF] Saving service mgr spec with placement smithi089=y;smithi134=x;count:2 2024-07-15T06:39:46.125 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:45 smithi089 bash[21793]: cephadm 2024-07-15T06:39:44.480741+0000 mgr.y (mgr.14150) 77 : cephadm [INF] Deploying daemon mgr.x on smithi134 2024-07-15T06:39:46.125 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:45 smithi089 bash[21793]: cluster 2024-07-15T06:39:44.871571+0000 mgr.y (mgr.14150) 78 : cluster [DBG] pgmap v43: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:46.157 INFO:teuthology.orchestra.run.smithi134.stderr:1+0 records in 2024-07-15T06:39:46.157 INFO:teuthology.orchestra.run.smithi134.stderr:1+0 records out 2024-07-15T06:39:46.157 INFO:teuthology.orchestra.run.smithi134.stderr:512 bytes copied, 0.000313312 s, 1.6 MB/s 2024-07-15T06:39:46.158 DEBUG:teuthology.orchestra.run.smithi134:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-07-15T06:39:46.205 DEBUG:teuthology.orchestra.run.smithi134:> stat /dev/vg_nvme/lv_2 2024-07-15T06:39:46.249 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:45 smithi134 bash[24068]: cephadm 2024-07-15T06:39:44.444017+0000 mgr.y (mgr.14150) 76 : cephadm [INF] Saving service mgr spec with placement smithi089=y;smithi134=x;count:2 2024-07-15T06:39:46.249 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:45 smithi134 bash[24068]: cephadm 2024-07-15T06:39:44.480741+0000 mgr.y (mgr.14150) 77 : cephadm [INF] Deploying daemon mgr.x on smithi134 2024-07-15T06:39:46.249 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:45 smithi134 bash[24068]: cluster 2024-07-15T06:39:44.871571+0000 mgr.y (mgr.14150) 78 : cluster [DBG] pgmap v43: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:46.253 INFO:teuthology.orchestra.run.smithi134.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-07-15T06:39:46.253 INFO:teuthology.orchestra.run.smithi134.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-07-15T06:39:46.253 INFO:teuthology.orchestra.run.smithi134.stdout:Device: 5h/5d Inode: 887 Links: 1 2024-07-15T06:39:46.253 INFO:teuthology.orchestra.run.smithi134.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-07-15T06:39:46.253 INFO:teuthology.orchestra.run.smithi134.stdout:Access: 2024-07-15 06:30:44.903676218 +0000 2024-07-15T06:39:46.253 INFO:teuthology.orchestra.run.smithi134.stdout:Modify: 2024-07-15 06:30:44.635678609 +0000 2024-07-15T06:39:46.253 INFO:teuthology.orchestra.run.smithi134.stdout:Change: 2024-07-15 06:30:44.635678609 +0000 2024-07-15T06:39:46.253 INFO:teuthology.orchestra.run.smithi134.stdout: Birth: - 2024-07-15T06:39:46.253 DEBUG:teuthology.orchestra.run.smithi134:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-07-15T06:39:46.306 INFO:teuthology.orchestra.run.smithi134.stderr:1+0 records in 2024-07-15T06:39:46.306 INFO:teuthology.orchestra.run.smithi134.stderr:1+0 records out 2024-07-15T06:39:46.306 INFO:teuthology.orchestra.run.smithi134.stderr:512 bytes copied, 0.000310529 s, 1.6 MB/s 2024-07-15T06:39:46.307 DEBUG:teuthology.orchestra.run.smithi134:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-07-15T06:39:46.354 DEBUG:teuthology.orchestra.run.smithi134:> stat /dev/vg_nvme/lv_3 2024-07-15T06:39:46.401 INFO:teuthology.orchestra.run.smithi134.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-07-15T06:39:46.401 INFO:teuthology.orchestra.run.smithi134.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-07-15T06:39:46.402 INFO:teuthology.orchestra.run.smithi134.stdout:Device: 5h/5d Inode: 905 Links: 1 2024-07-15T06:39:46.402 INFO:teuthology.orchestra.run.smithi134.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-07-15T06:39:46.402 INFO:teuthology.orchestra.run.smithi134.stdout:Access: 2024-07-15 06:30:45.419671615 +0000 2024-07-15T06:39:46.402 INFO:teuthology.orchestra.run.smithi134.stdout:Modify: 2024-07-15 06:30:45.155673969 +0000 2024-07-15T06:39:46.402 INFO:teuthology.orchestra.run.smithi134.stdout:Change: 2024-07-15 06:30:45.155673969 +0000 2024-07-15T06:39:46.402 INFO:teuthology.orchestra.run.smithi134.stdout: Birth: - 2024-07-15T06:39:46.402 DEBUG:teuthology.orchestra.run.smithi134:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-07-15T06:39:46.456 INFO:teuthology.orchestra.run.smithi134.stderr:1+0 records in 2024-07-15T06:39:46.456 INFO:teuthology.orchestra.run.smithi134.stderr:1+0 records out 2024-07-15T06:39:46.456 INFO:teuthology.orchestra.run.smithi134.stderr:512 bytes copied, 0.000419298 s, 1.2 MB/s 2024-07-15T06:39:46.457 DEBUG:teuthology.orchestra.run.smithi134:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-07-15T06:39:46.507 DEBUG:teuthology.orchestra.run.smithi134:> stat /dev/vg_nvme/lv_4 2024-07-15T06:39:46.553 INFO:teuthology.orchestra.run.smithi134.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-07-15T06:39:46.554 INFO:teuthology.orchestra.run.smithi134.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-07-15T06:39:46.554 INFO:teuthology.orchestra.run.smithi134.stdout:Device: 5h/5d Inode: 916 Links: 1 2024-07-15T06:39:46.554 INFO:teuthology.orchestra.run.smithi134.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-07-15T06:39:46.554 INFO:teuthology.orchestra.run.smithi134.stdout:Access: 2024-07-15 06:30:45.879667511 +0000 2024-07-15T06:39:46.554 INFO:teuthology.orchestra.run.smithi134.stdout:Modify: 2024-07-15 06:30:45.631669724 +0000 2024-07-15T06:39:46.554 INFO:teuthology.orchestra.run.smithi134.stdout:Change: 2024-07-15 06:30:45.631669724 +0000 2024-07-15T06:39:46.555 INFO:teuthology.orchestra.run.smithi134.stdout: Birth: - 2024-07-15T06:39:46.555 DEBUG:teuthology.orchestra.run.smithi134:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-07-15T06:39:46.608 INFO:teuthology.orchestra.run.smithi134.stderr:1+0 records in 2024-07-15T06:39:46.608 INFO:teuthology.orchestra.run.smithi134.stderr:1+0 records out 2024-07-15T06:39:46.608 INFO:teuthology.orchestra.run.smithi134.stderr:512 bytes copied, 0.000391722 s, 1.3 MB/s 2024-07-15T06:39:46.609 DEBUG:teuthology.orchestra.run.smithi134:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-07-15T06:39:46.659 INFO:tasks.cephadm:Deploying osd.0 on smithi089 with /dev/vg_nvme/lv_4... 2024-07-15T06:39:46.659 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-07-15T06:39:47.167 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:47 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:39:47.167 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:39:47 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:39:47.168 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:39:47 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:39:47.168 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:39:47 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:39:47.168 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:39:47 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:39:47.560 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:39:47 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:39:47.560 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:39:47 smithi134 systemd[1]: Started Ceph mgr.x for 75ba02bc-4274-11ef-bcad-c7b262605968. 2024-07-15T06:39:47.560 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:47 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:39:48.761 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:48 smithi134 bash[24068]: cluster 2024-07-15T06:39:46.871841+0000 mgr.y (mgr.14150) 79 : cluster [DBG] pgmap v44: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:48.761 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:48 smithi134 bash[24068]: audit 2024-07-15T06:39:47.395356+0000 mon.a (mon.0) 223 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:48.761 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:48 smithi134 bash[24068]: audit 2024-07-15T06:39:47.404743+0000 mon.a (mon.0) 224 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:48.761 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:48 smithi134 bash[24068]: audit 2024-07-15T06:39:47.414678+0000 mon.a (mon.0) 225 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:48.762 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:48 smithi134 bash[24068]: audit 2024-07-15T06:39:47.423801+0000 mon.a (mon.0) 226 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:48.762 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:48 smithi134 bash[24068]: audit 2024-07-15T06:39:47.481312+0000 mon.a (mon.0) 227 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:39:48.762 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:48 smithi134 bash[24068]: audit 2024-07-15T06:39:48.010881+0000 mon.a (mon.0) 228 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:48.875 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:48 smithi089 bash[21793]: cluster 2024-07-15T06:39:46.871841+0000 mgr.y (mgr.14150) 79 : cluster [DBG] pgmap v44: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:48.875 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:48 smithi089 bash[21793]: audit 2024-07-15T06:39:47.395356+0000 mon.a (mon.0) 223 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:48.875 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:48 smithi089 bash[21793]: audit 2024-07-15T06:39:47.404743+0000 mon.a (mon.0) 224 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:48.875 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:48 smithi089 bash[21793]: audit 2024-07-15T06:39:47.414678+0000 mon.a (mon.0) 225 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:48.875 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:48 smithi089 bash[21793]: audit 2024-07-15T06:39:47.423801+0000 mon.a (mon.0) 226 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:48.875 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:48 smithi089 bash[21793]: audit 2024-07-15T06:39:47.481312+0000 mon.a (mon.0) 227 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:39:48.875 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:48 smithi089 bash[21793]: audit 2024-07-15T06:39:48.010881+0000 mon.a (mon.0) 228 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:48.875 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:48 smithi089 bash[31372]: cluster 2024-07-15T06:39:46.871841+0000 mgr.y (mgr.14150) 79 : cluster [DBG] pgmap v44: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:48.875 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:48 smithi089 bash[31372]: audit 2024-07-15T06:39:47.395356+0000 mon.a (mon.0) 223 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:48.875 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:48 smithi089 bash[31372]: audit 2024-07-15T06:39:47.404743+0000 mon.a (mon.0) 224 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:48.876 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:48 smithi089 bash[31372]: audit 2024-07-15T06:39:47.414678+0000 mon.a (mon.0) 225 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:48.876 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:48 smithi089 bash[31372]: audit 2024-07-15T06:39:47.423801+0000 mon.a (mon.0) 226 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:48.876 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:48 smithi089 bash[31372]: audit 2024-07-15T06:39:47.481312+0000 mon.a (mon.0) 227 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:39:48.876 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:48 smithi089 bash[31372]: audit 2024-07-15T06:39:48.010881+0000 mon.a (mon.0) 228 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:49.059 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:39:48 smithi134 bash[25019]: debug 2024-07-15T06:39:48.862+0000 7fee22950640 1 -- 172.21.15.134:0/441541754 <== mon.1 v2:172.21.15.134:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x562ec9e1e9c0 con 0x562ec9e2a800 2024-07-15T06:39:49.060 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:39:48 smithi134 bash[25019]: debug 2024-07-15T06:39:48.978+0000 7fee251b8ec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-07-15T06:39:50.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:50 smithi134 bash[24068]: cluster 2024-07-15T06:39:48.872122+0000 mgr.y (mgr.14150) 80 : cluster [DBG] pgmap v45: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:50.560 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:39:50 smithi134 bash[25019]: debug 2024-07-15T06:39:50.182+0000 7fee251b8ec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-07-15T06:39:50.560 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:39:50 smithi134 bash[25019]: debug 2024-07-15T06:39:50.246+0000 7fee251b8ec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-07-15T06:39:50.560 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:39:50 smithi134 bash[25019]: debug 2024-07-15T06:39:50.398+0000 7fee251b8ec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-07-15T06:39:50.874 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:50 smithi089 bash[21793]: cluster 2024-07-15T06:39:48.872122+0000 mgr.y (mgr.14150) 80 : cluster [DBG] pgmap v45: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:50.875 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:50 smithi089 bash[31372]: cluster 2024-07-15T06:39:48.872122+0000 mgr.y (mgr.14150) 80 : cluster [DBG] pgmap v45: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:51.309 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:39:51 smithi134 bash[25019]: debug 2024-07-15T06:39:51.038+0000 7fee251b8ec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-07-15T06:39:51.309 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:39:51 smithi134 bash[25019]: debug 2024-07-15T06:39:51.106+0000 7fee251b8ec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-07-15T06:39:51.309 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:39:51 smithi134 bash[25019]: debug 2024-07-15T06:39:51.174+0000 7fee251b8ec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-07-15T06:39:51.377 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T06:39:51.739 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:39:51 smithi134 bash[25019]: debug 2024-07-15T06:39:51.342+0000 7fee251b8ec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-07-15T06:39:51.739 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:39:51 smithi134 bash[25019]: debug 2024-07-15T06:39:51.426+0000 7fee251b8ec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-07-15T06:39:51.739 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:39:51 smithi134 bash[25019]: debug 2024-07-15T06:39:51.494+0000 7fee251b8ec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-07-15T06:39:52.054 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:39:51 smithi134 bash[25019]: debug 2024-07-15T06:39:51.734+0000 7fee251b8ec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-07-15T06:39:52.054 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:39:51 smithi134 bash[25019]: debug 2024-07-15T06:39:51.802+0000 7fee251b8ec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-07-15T06:39:52.054 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:39:51 smithi134 bash[25019]: debug 2024-07-15T06:39:51.974+0000 7fee251b8ec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-07-15T06:39:52.054 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:39:52 smithi134 bash[25019]: debug 2024-07-15T06:39:52.050+0000 7fee251b8ec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-07-15T06:39:52.309 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:39:52 smithi134 bash[25019]: debug 2024-07-15T06:39:52.286+0000 7fee251b8ec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-07-15T06:39:52.684 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:39:52 smithi134 bash[25019]: debug 2024-07-15T06:39:52.446+0000 7fee251b8ec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-07-15T06:39:52.684 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:39:52 smithi134 bash[25019]: debug 2024-07-15T06:39:52.526+0000 7fee251b8ec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-07-15T06:39:52.684 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:39:52 smithi134 bash[25019]: debug 2024-07-15T06:39:52.678+0000 7fee251b8ec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-07-15T06:39:52.684 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:52 smithi134 bash[24068]: cluster 2024-07-15T06:39:50.872449+0000 mgr.y (mgr.14150) 81 : cluster [DBG] pgmap v46: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:52.684 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:52 smithi134 bash[24068]: audit 2024-07-15T06:39:51.425505+0000 mon.a (mon.0) 229 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:52.684 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:52 smithi134 bash[24068]: audit 2024-07-15T06:39:51.431299+0000 mon.a (mon.0) 230 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:52.684 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:52 smithi134 bash[24068]: audit 2024-07-15T06:39:51.814624+0000 mon.a (mon.0) 231 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:39:52.684 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:52 smithi134 bash[24068]: audit 2024-07-15T06:39:51.816400+0000 mon.a (mon.0) 232 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:39:52.684 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:52 smithi134 bash[24068]: audit 2024-07-15T06:39:51.827071+0000 mon.a (mon.0) 233 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:52.685 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:52 smithi134 bash[24068]: audit 2024-07-15T06:39:51.867552+0000 mon.a (mon.0) 234 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-07-15T06:39:52.685 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:52 smithi134 bash[24068]: audit 2024-07-15T06:39:51.868972+0000 mon.a (mon.0) 235 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-07-15T06:39:52.685 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:52 smithi134 bash[24068]: audit 2024-07-15T06:39:51.870154+0000 mon.a (mon.0) 236 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:39:52.762 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:52 smithi089 bash[31372]: cluster 2024-07-15T06:39:50.872449+0000 mgr.y (mgr.14150) 81 : cluster [DBG] pgmap v46: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:52.762 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:52 smithi089 bash[31372]: audit 2024-07-15T06:39:51.425505+0000 mon.a (mon.0) 229 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:52.762 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:52 smithi089 bash[31372]: audit 2024-07-15T06:39:51.431299+0000 mon.a (mon.0) 230 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:52.762 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:52 smithi089 bash[31372]: audit 2024-07-15T06:39:51.814624+0000 mon.a (mon.0) 231 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:39:52.762 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:52 smithi089 bash[31372]: audit 2024-07-15T06:39:51.816400+0000 mon.a (mon.0) 232 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:39:52.762 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:52 smithi089 bash[31372]: audit 2024-07-15T06:39:51.827071+0000 mon.a (mon.0) 233 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:52.762 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:52 smithi089 bash[31372]: audit 2024-07-15T06:39:51.867552+0000 mon.a (mon.0) 234 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-07-15T06:39:52.762 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:52 smithi089 bash[31372]: audit 2024-07-15T06:39:51.868972+0000 mon.a (mon.0) 235 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-07-15T06:39:52.762 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:52 smithi089 bash[31372]: audit 2024-07-15T06:39:51.870154+0000 mon.a (mon.0) 236 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:39:52.763 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:52 smithi089 bash[21793]: cluster 2024-07-15T06:39:50.872449+0000 mgr.y (mgr.14150) 81 : cluster [DBG] pgmap v46: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:52.763 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:52 smithi089 bash[21793]: audit 2024-07-15T06:39:51.425505+0000 mon.a (mon.0) 229 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:52.763 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:52 smithi089 bash[21793]: audit 2024-07-15T06:39:51.431299+0000 mon.a (mon.0) 230 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:52.763 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:52 smithi089 bash[21793]: audit 2024-07-15T06:39:51.814624+0000 mon.a (mon.0) 231 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:39:52.763 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:52 smithi089 bash[21793]: audit 2024-07-15T06:39:51.816400+0000 mon.a (mon.0) 232 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:39:52.763 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:52 smithi089 bash[21793]: audit 2024-07-15T06:39:51.827071+0000 mon.a (mon.0) 233 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:52.763 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:52 smithi089 bash[21793]: audit 2024-07-15T06:39:51.867552+0000 mon.a (mon.0) 234 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-07-15T06:39:52.763 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:52 smithi089 bash[21793]: audit 2024-07-15T06:39:51.868972+0000 mon.a (mon.0) 235 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-07-15T06:39:52.763 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:52 smithi089 bash[21793]: audit 2024-07-15T06:39:51.870154+0000 mon.a (mon.0) 236 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:39:53.707 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:53 smithi089 bash[21793]: cephadm 2024-07-15T06:39:51.866982+0000 mgr.y (mgr.14150) 82 : cephadm [INF] Reconfiguring mgr.y (unknown last config time)... 2024-07-15T06:39:53.707 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:53 smithi089 bash[21793]: cephadm 2024-07-15T06:39:51.871215+0000 mgr.y (mgr.14150) 83 : cephadm [INF] Reconfiguring daemon mgr.y on smithi089 2024-07-15T06:39:53.707 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:53 smithi089 bash[31372]: cephadm 2024-07-15T06:39:51.866982+0000 mgr.y (mgr.14150) 82 : cephadm [INF] Reconfiguring mgr.y (unknown last config time)... 2024-07-15T06:39:53.707 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:53 smithi089 bash[31372]: cephadm 2024-07-15T06:39:51.871215+0000 mgr.y (mgr.14150) 83 : cephadm [INF] Reconfiguring daemon mgr.y on smithi089 2024-07-15T06:39:53.726 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:53 smithi134 bash[24068]: cephadm 2024-07-15T06:39:51.866982+0000 mgr.y (mgr.14150) 82 : cephadm [INF] Reconfiguring mgr.y (unknown last config time)... 2024-07-15T06:39:53.726 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:53 smithi134 bash[24068]: cephadm 2024-07-15T06:39:51.871215+0000 mgr.y (mgr.14150) 83 : cephadm [INF] Reconfiguring daemon mgr.y on smithi089 2024-07-15T06:39:53.726 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:39:53 smithi134 bash[25019]: debug 2024-07-15T06:39:53.446+0000 7fee251b8ec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-07-15T06:39:53.726 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:39:53 smithi134 bash[25019]: debug 2024-07-15T06:39:53.522+0000 7fee251b8ec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-07-15T06:39:54.059 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:39:53 smithi134 bash[25019]: debug 2024-07-15T06:39:53.722+0000 7fee251b8ec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-07-15T06:39:54.060 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:39:53 smithi134 bash[25019]: debug 2024-07-15T06:39:53.794+0000 7fee251b8ec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-07-15T06:39:54.435 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:39:54 smithi134 bash[25019]: debug 2024-07-15T06:39:54.110+0000 7fee251b8ec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-07-15T06:39:54.435 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:39:54 smithi134 bash[25019]: debug 2024-07-15T06:39:54.190+0000 7fee251b8ec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-07-15T06:39:54.625 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:54 smithi089 bash[21793]: cluster 2024-07-15T06:39:52.872847+0000 mgr.y (mgr.14150) 84 : cluster [DBG] pgmap v47: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:54.625 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:54 smithi089 bash[21793]: cluster 2024-07-15T06:39:54.216958+0000 mon.a (mon.0) 237 : cluster [DBG] Standby manager daemon x started 2024-07-15T06:39:54.625 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:54 smithi089 bash[21793]: audit 2024-07-15T06:39:54.217804+0000 mon.b (mon.1) 3 : audit [DBG] from='mgr.? 172.21.15.134:0/1680447747' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-07-15T06:39:54.625 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:54 smithi089 bash[21793]: audit 2024-07-15T06:39:54.218593+0000 mon.b (mon.1) 4 : audit [DBG] from='mgr.? 172.21.15.134:0/1680447747' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-07-15T06:39:54.625 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:54 smithi089 bash[21793]: audit 2024-07-15T06:39:54.219653+0000 mon.b (mon.1) 5 : audit [DBG] from='mgr.? 172.21.15.134:0/1680447747' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-07-15T06:39:54.625 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:54 smithi089 bash[21793]: audit 2024-07-15T06:39:54.220095+0000 mon.b (mon.1) 6 : audit [DBG] from='mgr.? 172.21.15.134:0/1680447747' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-07-15T06:39:54.626 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:54 smithi089 bash[31372]: cluster 2024-07-15T06:39:52.872847+0000 mgr.y (mgr.14150) 84 : cluster [DBG] pgmap v47: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:54.626 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:54 smithi089 bash[31372]: cluster 2024-07-15T06:39:54.216958+0000 mon.a (mon.0) 237 : cluster [DBG] Standby manager daemon x started 2024-07-15T06:39:54.626 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:54 smithi089 bash[31372]: audit 2024-07-15T06:39:54.217804+0000 mon.b (mon.1) 3 : audit [DBG] from='mgr.? 172.21.15.134:0/1680447747' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-07-15T06:39:54.626 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:54 smithi089 bash[31372]: audit 2024-07-15T06:39:54.218593+0000 mon.b (mon.1) 4 : audit [DBG] from='mgr.? 172.21.15.134:0/1680447747' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-07-15T06:39:54.626 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:54 smithi089 bash[31372]: audit 2024-07-15T06:39:54.219653+0000 mon.b (mon.1) 5 : audit [DBG] from='mgr.? 172.21.15.134:0/1680447747' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-07-15T06:39:54.626 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:54 smithi089 bash[31372]: audit 2024-07-15T06:39:54.220095+0000 mon.b (mon.1) 6 : audit [DBG] from='mgr.? 172.21.15.134:0/1680447747' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-07-15T06:39:54.810 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:54 smithi134 bash[24068]: cluster 2024-07-15T06:39:52.872847+0000 mgr.y (mgr.14150) 84 : cluster [DBG] pgmap v47: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:54.810 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:54 smithi134 bash[24068]: cluster 2024-07-15T06:39:54.216958+0000 mon.a (mon.0) 237 : cluster [DBG] Standby manager daemon x started 2024-07-15T06:39:54.810 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:54 smithi134 bash[24068]: audit 2024-07-15T06:39:54.217804+0000 mon.b (mon.1) 3 : audit [DBG] from='mgr.? 172.21.15.134:0/1680447747' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-07-15T06:39:54.810 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:54 smithi134 bash[24068]: audit 2024-07-15T06:39:54.218593+0000 mon.b (mon.1) 4 : audit [DBG] from='mgr.? 172.21.15.134:0/1680447747' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-07-15T06:39:54.810 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:54 smithi134 bash[24068]: audit 2024-07-15T06:39:54.219653+0000 mon.b (mon.1) 5 : audit [DBG] from='mgr.? 172.21.15.134:0/1680447747' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-07-15T06:39:54.810 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:54 smithi134 bash[24068]: audit 2024-07-15T06:39:54.220095+0000 mon.b (mon.1) 6 : audit [DBG] from='mgr.? 172.21.15.134:0/1680447747' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-07-15T06:39:55.566 INFO:teuthology.orchestra.run.smithi089.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-07-15T06:39:55.625 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:55 smithi089 bash[21793]: cluster 2024-07-15T06:39:54.452134+0000 mon.a (mon.0) 238 : cluster [DBG] mgrmap e14: y(active, since 107s), standbys: x 2024-07-15T06:39:55.625 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:55 smithi089 bash[21793]: audit 2024-07-15T06:39:54.452407+0000 mon.a (mon.0) 239 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-07-15T06:39:55.625 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:55 smithi089 bash[21793]: audit 2024-07-15T06:39:54.735079+0000 mon.a (mon.0) 240 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:55.625 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:55 smithi089 bash[21793]: audit 2024-07-15T06:39:54.741747+0000 mon.a (mon.0) 241 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:55.625 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:55 smithi089 bash[21793]: audit 2024-07-15T06:39:54.743958+0000 mon.a (mon.0) 242 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:39:55.625 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:55 smithi089 bash[21793]: audit 2024-07-15T06:39:54.746400+0000 mon.a (mon.0) 243 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:39:55.625 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:55 smithi089 bash[21793]: audit 2024-07-15T06:39:54.747800+0000 mon.a (mon.0) 244 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:39:55.626 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:55 smithi089 bash[21793]: audit 2024-07-15T06:39:54.756524+0000 mon.a (mon.0) 245 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:55.626 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:55 smithi089 bash[31372]: cluster 2024-07-15T06:39:54.452134+0000 mon.a (mon.0) 238 : cluster [DBG] mgrmap e14: y(active, since 107s), standbys: x 2024-07-15T06:39:55.626 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:55 smithi089 bash[31372]: audit 2024-07-15T06:39:54.452407+0000 mon.a (mon.0) 239 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-07-15T06:39:55.626 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:55 smithi089 bash[31372]: audit 2024-07-15T06:39:54.735079+0000 mon.a (mon.0) 240 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:55.626 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:55 smithi089 bash[31372]: audit 2024-07-15T06:39:54.741747+0000 mon.a (mon.0) 241 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:55.626 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:55 smithi089 bash[31372]: audit 2024-07-15T06:39:54.743958+0000 mon.a (mon.0) 242 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:39:55.626 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:55 smithi089 bash[31372]: audit 2024-07-15T06:39:54.746400+0000 mon.a (mon.0) 243 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:39:55.626 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:55 smithi089 bash[31372]: audit 2024-07-15T06:39:54.747800+0000 mon.a (mon.0) 244 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:39:55.626 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:55 smithi089 bash[31372]: audit 2024-07-15T06:39:54.756524+0000 mon.a (mon.0) 245 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:55.770 INFO:teuthology.orchestra.run.smithi089.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-07-15T06:39:55.786 INFO:teuthology.orchestra.run.smithi089.stderr: stderr: 10+0 records in 2024-07-15T06:39:55.786 INFO:teuthology.orchestra.run.smithi089.stderr:10+0 records out 2024-07-15T06:39:55.786 INFO:teuthology.orchestra.run.smithi089.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0140344 s, 747 MB/s 2024-07-15T06:39:55.786 INFO:teuthology.orchestra.run.smithi089.stderr:--> Zapping successful for: 2024-07-15T06:39:55.810 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:55 smithi134 bash[24068]: cluster 2024-07-15T06:39:54.452134+0000 mon.a (mon.0) 238 : cluster [DBG] mgrmap e14: y(active, since 107s), standbys: x 2024-07-15T06:39:55.810 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:55 smithi134 bash[24068]: audit 2024-07-15T06:39:54.452407+0000 mon.a (mon.0) 239 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-07-15T06:39:55.810 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:55 smithi134 bash[24068]: audit 2024-07-15T06:39:54.735079+0000 mon.a (mon.0) 240 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:55.810 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:55 smithi134 bash[24068]: audit 2024-07-15T06:39:54.741747+0000 mon.a (mon.0) 241 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:55.810 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:55 smithi134 bash[24068]: audit 2024-07-15T06:39:54.743958+0000 mon.a (mon.0) 242 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:39:55.810 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:55 smithi134 bash[24068]: audit 2024-07-15T06:39:54.746400+0000 mon.a (mon.0) 243 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:39:55.810 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:55 smithi134 bash[24068]: audit 2024-07-15T06:39:54.747800+0000 mon.a (mon.0) 244 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:39:55.810 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:55 smithi134 bash[24068]: audit 2024-07-15T06:39:54.756524+0000 mon.a (mon.0) 245 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:39:56.415 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph orch daemon add osd smithi089:vg_nvme/lv_4 2024-07-15T06:39:56.809 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:56 smithi134 bash[24068]: cluster 2024-07-15T06:39:54.873199+0000 mgr.y (mgr.14150) 85 : cluster [DBG] pgmap v48: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:56.875 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:56 smithi089 bash[21793]: cluster 2024-07-15T06:39:54.873199+0000 mgr.y (mgr.14150) 85 : cluster [DBG] pgmap v48: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:56.875 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:56 smithi089 bash[31372]: cluster 2024-07-15T06:39:54.873199+0000 mgr.y (mgr.14150) 85 : cluster [DBG] pgmap v48: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:57.809 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:57 smithi134 bash[24068]: cluster 2024-07-15T06:39:56.873645+0000 mgr.y (mgr.14150) 86 : cluster [DBG] pgmap v49: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:57.875 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:57 smithi089 bash[21793]: cluster 2024-07-15T06:39:56.873645+0000 mgr.y (mgr.14150) 86 : cluster [DBG] pgmap v49: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:39:57.875 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:57 smithi089 bash[31372]: cluster 2024-07-15T06:39:56.873645+0000 mgr.y (mgr.14150) 86 : cluster [DBG] pgmap v49: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:40:00.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:39:59 smithi134 bash[24068]: cluster 2024-07-15T06:39:58.874091+0000 mgr.y (mgr.14150) 87 : cluster [DBG] pgmap v50: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:40:00.375 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:39:59 smithi089 bash[21793]: cluster 2024-07-15T06:39:58.874091+0000 mgr.y (mgr.14150) 87 : cluster [DBG] pgmap v50: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:40:00.375 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:39:59 smithi089 bash[31372]: cluster 2024-07-15T06:39:58.874091+0000 mgr.y (mgr.14150) 87 : cluster [DBG] pgmap v50: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:40:01.137 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T06:40:01.241 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:40:00 smithi089 bash[31372]: cluster 2024-07-15T06:40:00.000148+0000 mon.a (mon.0) 246 : cluster [INF] overall HEALTH_OK 2024-07-15T06:40:01.241 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:40:00 smithi089 bash[21793]: cluster 2024-07-15T06:40:00.000148+0000 mon.a (mon.0) 246 : cluster [INF] overall HEALTH_OK 2024-07-15T06:40:01.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:40:00 smithi134 bash[24068]: cluster 2024-07-15T06:40:00.000148+0000 mon.a (mon.0) 246 : cluster [INF] overall HEALTH_OK 2024-07-15T06:40:02.229 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:40:01 smithi089 bash[21793]: cluster 2024-07-15T06:40:00.874532+0000 mgr.y (mgr.14150) 88 : cluster [DBG] pgmap v51: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:40:02.229 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:40:01 smithi089 bash[31372]: cluster 2024-07-15T06:40:00.874532+0000 mgr.y (mgr.14150) 88 : cluster [DBG] pgmap v51: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:40:02.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:40:01 smithi134 bash[24068]: cluster 2024-07-15T06:40:00.874532+0000 mgr.y (mgr.14150) 88 : cluster [DBG] pgmap v51: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:40:04.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:40:03 smithi134 bash[24068]: cluster 2024-07-15T06:40:02.874936+0000 mgr.y (mgr.14150) 89 : cluster [DBG] pgmap v52: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:40:04.374 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:40:03 smithi089 bash[21793]: cluster 2024-07-15T06:40:02.874936+0000 mgr.y (mgr.14150) 89 : cluster [DBG] pgmap v52: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:40:04.375 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:40:03 smithi089 bash[31372]: cluster 2024-07-15T06:40:02.874936+0000 mgr.y (mgr.14150) 89 : cluster [DBG] pgmap v52: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:40:05.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:40:04 smithi134 bash[24068]: audit 2024-07-15T06:40:04.311211+0000 mgr.y (mgr.14150) 90 : audit [DBG] from='client.24113 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi089:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T06:40:05.310 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:40:04 smithi134 bash[24068]: audit 2024-07-15T06:40:04.318544+0000 mon.a (mon.0) 247 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-07-15T06:40:05.310 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:40:04 smithi134 bash[24068]: audit 2024-07-15T06:40:04.321608+0000 mon.a (mon.0) 248 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-07-15T06:40:05.310 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:40:04 smithi134 bash[24068]: audit 2024-07-15T06:40:04.322728+0000 mon.a (mon.0) 249 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:40:05.375 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:40:04 smithi089 bash[31372]: audit 2024-07-15T06:40:04.311211+0000 mgr.y (mgr.14150) 90 : audit [DBG] from='client.24113 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi089:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T06:40:05.375 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:40:04 smithi089 bash[31372]: audit 2024-07-15T06:40:04.318544+0000 mon.a (mon.0) 247 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-07-15T06:40:05.375 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:40:04 smithi089 bash[31372]: audit 2024-07-15T06:40:04.321608+0000 mon.a (mon.0) 248 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-07-15T06:40:05.375 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:40:04 smithi089 bash[31372]: audit 2024-07-15T06:40:04.322728+0000 mon.a (mon.0) 249 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:40:05.375 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:40:04 smithi089 bash[21793]: audit 2024-07-15T06:40:04.311211+0000 mgr.y (mgr.14150) 90 : audit [DBG] from='client.24113 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi089:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T06:40:05.375 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:40:04 smithi089 bash[21793]: audit 2024-07-15T06:40:04.318544+0000 mon.a (mon.0) 247 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-07-15T06:40:05.375 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:40:04 smithi089 bash[21793]: audit 2024-07-15T06:40:04.321608+0000 mon.a (mon.0) 248 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-07-15T06:40:05.375 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:40:04 smithi089 bash[21793]: audit 2024-07-15T06:40:04.322728+0000 mon.a (mon.0) 249 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:40:06.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:40:05 smithi134 bash[24068]: cluster 2024-07-15T06:40:04.875470+0000 mgr.y (mgr.14150) 91 : cluster [DBG] pgmap v53: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:40:06.374 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:40:05 smithi089 bash[31372]: cluster 2024-07-15T06:40:04.875470+0000 mgr.y (mgr.14150) 91 : cluster [DBG] pgmap v53: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:40:06.375 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:40:05 smithi089 bash[21793]: cluster 2024-07-15T06:40:04.875470+0000 mgr.y (mgr.14150) 91 : cluster [DBG] pgmap v53: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:40:08.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:40:07 smithi134 bash[24068]: cluster 2024-07-15T06:40:06.875959+0000 mgr.y (mgr.14150) 92 : cluster [DBG] pgmap v54: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:40:08.374 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:40:07 smithi089 bash[21793]: cluster 2024-07-15T06:40:06.875959+0000 mgr.y (mgr.14150) 92 : cluster [DBG] pgmap v54: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:40:08.375 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:40:07 smithi089 bash[31372]: cluster 2024-07-15T06:40:06.875959+0000 mgr.y (mgr.14150) 92 : cluster [DBG] pgmap v54: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:40:10.286 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:40:09 smithi089 bash[21793]: cluster 2024-07-15T06:40:08.876398+0000 mgr.y (mgr.14150) 93 : cluster [DBG] pgmap v55: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:40:10.286 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:40:09 smithi089 bash[31372]: cluster 2024-07-15T06:40:08.876398+0000 mgr.y (mgr.14150) 93 : cluster [DBG] pgmap v55: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:40:10.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:40:09 smithi134 bash[24068]: cluster 2024-07-15T06:40:08.876398+0000 mgr.y (mgr.14150) 93 : cluster [DBG] pgmap v55: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:40:12.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:40:11 smithi134 bash[24068]: cluster 2024-07-15T06:40:10.876851+0000 mgr.y (mgr.14150) 94 : cluster [DBG] pgmap v56: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:40:12.374 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:40:11 smithi089 bash[21793]: cluster 2024-07-15T06:40:10.876851+0000 mgr.y (mgr.14150) 94 : cluster [DBG] pgmap v56: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:40:12.375 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:40:11 smithi089 bash[31372]: cluster 2024-07-15T06:40:10.876851+0000 mgr.y (mgr.14150) 94 : cluster [DBG] pgmap v56: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:40:13.310 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:40:12 smithi134 bash[24068]: audit 2024-07-15T06:40:12.363439+0000 mon.c (mon.2) 2 : audit [INF] from='client.? 172.21.15.89:0/98066903' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "79357114-251f-489d-bef5-f6ec37002212"}]: dispatch 2024-07-15T06:40:13.310 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:40:12 smithi134 bash[24068]: audit 2024-07-15T06:40:12.363920+0000 mon.a (mon.0) 250 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "79357114-251f-489d-bef5-f6ec37002212"}]: dispatch 2024-07-15T06:40:13.310 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:40:12 smithi134 bash[24068]: audit 2024-07-15T06:40:12.370386+0000 mon.a (mon.0) 251 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "79357114-251f-489d-bef5-f6ec37002212"}]': finished 2024-07-15T06:40:13.310 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:40:12 smithi134 bash[24068]: cluster 2024-07-15T06:40:12.370541+0000 mon.a (mon.0) 252 : cluster [DBG] osdmap e5: 1 total, 0 up, 1 in 2024-07-15T06:40:13.310 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:40:12 smithi134 bash[24068]: audit 2024-07-15T06:40:12.370704+0000 mon.a (mon.0) 253 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-07-15T06:40:13.374 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:40:12 smithi089 bash[21793]: audit 2024-07-15T06:40:12.363439+0000 mon.c (mon.2) 2 : audit [INF] from='client.? 172.21.15.89:0/98066903' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "79357114-251f-489d-bef5-f6ec37002212"}]: dispatch 2024-07-15T06:40:13.375 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:40:12 smithi089 bash[21793]: audit 2024-07-15T06:40:12.363920+0000 mon.a (mon.0) 250 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "79357114-251f-489d-bef5-f6ec37002212"}]: dispatch 2024-07-15T06:40:13.375 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:40:12 smithi089 bash[21793]: audit 2024-07-15T06:40:12.370386+0000 mon.a (mon.0) 251 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "79357114-251f-489d-bef5-f6ec37002212"}]': finished 2024-07-15T06:40:13.375 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:40:12 smithi089 bash[21793]: cluster 2024-07-15T06:40:12.370541+0000 mon.a (mon.0) 252 : cluster [DBG] osdmap e5: 1 total, 0 up, 1 in 2024-07-15T06:40:13.375 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:40:12 smithi089 bash[21793]: audit 2024-07-15T06:40:12.370704+0000 mon.a (mon.0) 253 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-07-15T06:40:13.375 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:40:12 smithi089 bash[31372]: audit 2024-07-15T06:40:12.363439+0000 mon.c (mon.2) 2 : audit [INF] from='client.? 172.21.15.89:0/98066903' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "79357114-251f-489d-bef5-f6ec37002212"}]: dispatch 2024-07-15T06:40:13.375 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:40:12 smithi089 bash[31372]: audit 2024-07-15T06:40:12.363920+0000 mon.a (mon.0) 250 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "79357114-251f-489d-bef5-f6ec37002212"}]: dispatch 2024-07-15T06:40:13.375 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:40:12 smithi089 bash[31372]: audit 2024-07-15T06:40:12.370386+0000 mon.a (mon.0) 251 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "79357114-251f-489d-bef5-f6ec37002212"}]': finished 2024-07-15T06:40:13.375 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:40:12 smithi089 bash[31372]: cluster 2024-07-15T06:40:12.370541+0000 mon.a (mon.0) 252 : cluster [DBG] osdmap e5: 1 total, 0 up, 1 in 2024-07-15T06:40:13.375 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:40:12 smithi089 bash[31372]: audit 2024-07-15T06:40:12.370704+0000 mon.a (mon.0) 253 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-07-15T06:40:14.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:40:13 smithi134 bash[24068]: cluster 2024-07-15T06:40:12.877137+0000 mgr.y (mgr.14150) 95 : cluster [DBG] pgmap v58: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:40:14.310 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:40:13 smithi134 bash[24068]: audit 2024-07-15T06:40:13.003362+0000 mon.c (mon.2) 3 : audit [DBG] from='client.? 172.21.15.89:0/2709896914' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-07-15T06:40:14.374 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:40:13 smithi089 bash[21793]: cluster 2024-07-15T06:40:12.877137+0000 mgr.y (mgr.14150) 95 : cluster [DBG] pgmap v58: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:40:14.375 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:40:13 smithi089 bash[21793]: audit 2024-07-15T06:40:13.003362+0000 mon.c (mon.2) 3 : audit [DBG] from='client.? 172.21.15.89:0/2709896914' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-07-15T06:40:14.375 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:40:13 smithi089 bash[31372]: cluster 2024-07-15T06:40:12.877137+0000 mgr.y (mgr.14150) 95 : cluster [DBG] pgmap v58: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:40:14.375 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:40:13 smithi089 bash[31372]: audit 2024-07-15T06:40:13.003362+0000 mon.c (mon.2) 3 : audit [DBG] from='client.? 172.21.15.89:0/2709896914' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-07-15T06:40:16.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:40:15 smithi134 bash[24068]: cluster 2024-07-15T06:40:14.877540+0000 mgr.y (mgr.14150) 96 : cluster [DBG] pgmap v59: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:40:16.374 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:40:15 smithi089 bash[21793]: cluster 2024-07-15T06:40:14.877540+0000 mgr.y (mgr.14150) 96 : cluster [DBG] pgmap v59: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:40:16.375 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:40:15 smithi089 bash[31372]: cluster 2024-07-15T06:40:14.877540+0000 mgr.y (mgr.14150) 96 : cluster [DBG] pgmap v59: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:40:18.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:40:17 smithi134 bash[24068]: cluster 2024-07-15T06:40:16.877945+0000 mgr.y (mgr.14150) 97 : cluster [DBG] pgmap v60: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:40:18.374 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:40:17 smithi089 bash[21793]: cluster 2024-07-15T06:40:16.877945+0000 mgr.y (mgr.14150) 97 : cluster [DBG] pgmap v60: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:40:18.375 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:40:17 smithi089 bash[31372]: cluster 2024-07-15T06:40:16.877945+0000 mgr.y (mgr.14150) 97 : cluster [DBG] pgmap v60: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:40:20.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:40:19 smithi134 bash[24068]: cluster 2024-07-15T06:40:18.878376+0000 mgr.y (mgr.14150) 98 : cluster [DBG] pgmap v61: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:40:20.374 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:40:19 smithi089 bash[21793]: cluster 2024-07-15T06:40:18.878376+0000 mgr.y (mgr.14150) 98 : cluster [DBG] pgmap v61: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:40:20.375 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:40:19 smithi089 bash[31372]: cluster 2024-07-15T06:40:18.878376+0000 mgr.y (mgr.14150) 98 : cluster [DBG] pgmap v61: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:40:22.124 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:40:21 smithi089 bash[21793]: cluster 2024-07-15T06:40:20.878670+0000 mgr.y (mgr.14150) 99 : cluster [DBG] pgmap v62: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:40:22.125 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:40:21 smithi089 bash[31372]: cluster 2024-07-15T06:40:20.878670+0000 mgr.y (mgr.14150) 99 : cluster [DBG] pgmap v62: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:40:22.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:40:21 smithi134 bash[24068]: cluster 2024-07-15T06:40:20.878670+0000 mgr.y (mgr.14150) 99 : cluster [DBG] pgmap v62: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:40:24.124 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:40:23 smithi089 bash[21793]: cluster 2024-07-15T06:40:22.879060+0000 mgr.y (mgr.14150) 100 : cluster [DBG] pgmap v63: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:40:24.125 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:40:23 smithi089 bash[31372]: cluster 2024-07-15T06:40:22.879060+0000 mgr.y (mgr.14150) 100 : cluster [DBG] pgmap v63: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:40:24.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:40:23 smithi134 bash[24068]: cluster 2024-07-15T06:40:22.879060+0000 mgr.y (mgr.14150) 100 : cluster [DBG] pgmap v63: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:40:24.997 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:40:24 smithi089 bash[31372]: audit 2024-07-15T06:40:24.419232+0000 mon.a (mon.0) 254 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-07-15T06:40:24.997 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:40:24 smithi089 bash[31372]: audit 2024-07-15T06:40:24.421061+0000 mon.a (mon.0) 255 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:40:24.998 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:40:24 smithi089 bash[31372]: cephadm 2024-07-15T06:40:24.422634+0000 mgr.y (mgr.14150) 101 : cephadm [INF] Deploying daemon osd.0 on smithi089 2024-07-15T06:40:25.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:40:24 smithi134 bash[24068]: audit 2024-07-15T06:40:24.419232+0000 mon.a (mon.0) 254 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-07-15T06:40:25.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:40:24 smithi134 bash[24068]: audit 2024-07-15T06:40:24.421061+0000 mon.a (mon.0) 255 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:40:25.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:40:24 smithi134 bash[24068]: cephadm 2024-07-15T06:40:24.422634+0000 mgr.y (mgr.14150) 101 : cephadm [INF] Deploying daemon osd.0 on smithi089 2024-07-15T06:40:25.374 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:40:24 smithi089 bash[21793]: audit 2024-07-15T06:40:24.419232+0000 mon.a (mon.0) 254 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-07-15T06:40:25.374 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:40:24 smithi089 bash[21793]: audit 2024-07-15T06:40:24.421061+0000 mon.a (mon.0) 255 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:40:25.375 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:40:24 smithi089 bash[21793]: cephadm 2024-07-15T06:40:24.422634+0000 mgr.y (mgr.14150) 101 : cephadm [INF] Deploying daemon osd.0 on smithi089 2024-07-15T06:40:26.124 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:40:25 smithi089 bash[31372]: cluster 2024-07-15T06:40:24.879523+0000 mgr.y (mgr.14150) 102 : cluster [DBG] pgmap v64: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:40:26.125 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:40:25 smithi089 bash[21793]: cluster 2024-07-15T06:40:24.879523+0000 mgr.y (mgr.14150) 102 : cluster [DBG] pgmap v64: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:40:26.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:40:25 smithi134 bash[24068]: cluster 2024-07-15T06:40:24.879523+0000 mgr.y (mgr.14150) 102 : cluster [DBG] pgmap v64: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:40:28.125 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:40:28 smithi089 bash[21793]: cluster 2024-07-15T06:40:26.879888+0000 mgr.y (mgr.14150) 103 : cluster [DBG] pgmap v65: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:40:28.125 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:40:28 smithi089 bash[31372]: cluster 2024-07-15T06:40:26.879888+0000 mgr.y (mgr.14150) 103 : cluster [DBG] pgmap v65: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:40:28.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:40:28 smithi134 bash[24068]: cluster 2024-07-15T06:40:26.879888+0000 mgr.y (mgr.14150) 103 : cluster [DBG] pgmap v65: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:40:28.624 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:40:28 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:40:28.625 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:40:28 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:40:28.625 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:40:28 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:40:29.001 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:40:28 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:40:29.001 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:40:28 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:40:29.002 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:40:28 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:40:29.002 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:40:29 smithi089 bash[21793]: audit 2024-07-15T06:40:28.846273+0000 mon.a (mon.0) 256 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:40:29.002 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:40:29 smithi089 bash[21793]: audit 2024-07-15T06:40:28.850053+0000 mon.a (mon.0) 257 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:40:29.002 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:40:29 smithi089 bash[21793]: audit 2024-07-15T06:40:28.854266+0000 mon.a (mon.0) 258 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:40:29.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:40:29 smithi134 bash[24068]: audit 2024-07-15T06:40:28.846273+0000 mon.a (mon.0) 256 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:40:29.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:40:29 smithi134 bash[24068]: audit 2024-07-15T06:40:28.850053+0000 mon.a (mon.0) 257 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:40:29.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:40:29 smithi134 bash[24068]: audit 2024-07-15T06:40:28.854266+0000 mon.a (mon.0) 258 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:40:29.374 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:40:29 smithi089 bash[31372]: audit 2024-07-15T06:40:28.846273+0000 mon.a (mon.0) 256 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:40:29.374 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:40:29 smithi089 bash[31372]: audit 2024-07-15T06:40:28.850053+0000 mon.a (mon.0) 257 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:40:29.375 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:40:29 smithi089 bash[31372]: audit 2024-07-15T06:40:28.854266+0000 mon.a (mon.0) 258 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:40:30.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:40:30 smithi134 bash[24068]: cluster 2024-07-15T06:40:28.880141+0000 mgr.y (mgr.14150) 104 : cluster [DBG] pgmap v66: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:40:30.343 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:40:30 smithi089 bash[21793]: cluster 2024-07-15T06:40:28.880141+0000 mgr.y (mgr.14150) 104 : cluster [DBG] pgmap v66: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:40:30.343 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:40:30 smithi089 bash[31372]: cluster 2024-07-15T06:40:28.880141+0000 mgr.y (mgr.14150) 104 : cluster [DBG] pgmap v66: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:40:32.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:40:32 smithi134 bash[24068]: cluster 2024-07-15T06:40:30.880550+0000 mgr.y (mgr.14150) 105 : cluster [DBG] pgmap v67: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:40:32.374 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:40:32 smithi089 bash[21793]: cluster 2024-07-15T06:40:30.880550+0000 mgr.y (mgr.14150) 105 : cluster [DBG] pgmap v67: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:40:32.375 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:40:32 smithi089 bash[31372]: cluster 2024-07-15T06:40:30.880550+0000 mgr.y (mgr.14150) 105 : cluster [DBG] pgmap v67: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:40:34.011 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:40:34 smithi089 bash[21793]: cluster 2024-07-15T06:40:32.881003+0000 mgr.y (mgr.14150) 106 : cluster [DBG] pgmap v68: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:40:34.011 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:40:34 smithi089 bash[21793]: audit 2024-07-15T06:40:33.840058+0000 mon.a (mon.0) 259 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:40:34.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:40:34 smithi134 bash[24068]: cluster 2024-07-15T06:40:32.881003+0000 mgr.y (mgr.14150) 106 : cluster [DBG] pgmap v68: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:40:34.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:40:34 smithi134 bash[24068]: audit 2024-07-15T06:40:33.840058+0000 mon.a (mon.0) 259 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:40:34.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:40:34 smithi134 bash[24068]: audit 2024-07-15T06:40:33.844021+0000 mon.a (mon.0) 260 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:40:34.374 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:40:34 smithi089 bash[21793]: audit 2024-07-15T06:40:33.844021+0000 mon.a (mon.0) 260 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:40:34.374 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:40:34 smithi089 bash[31372]: cluster 2024-07-15T06:40:32.881003+0000 mgr.y (mgr.14150) 106 : cluster [DBG] pgmap v68: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:40:34.374 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:40:34 smithi089 bash[31372]: audit 2024-07-15T06:40:33.840058+0000 mon.a (mon.0) 259 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:40:34.375 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:40:34 smithi089 bash[31372]: audit 2024-07-15T06:40:33.844021+0000 mon.a (mon.0) 260 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:40:35.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:40:35 smithi134 bash[24068]: audit 2024-07-15T06:40:34.180801+0000 mon.a (mon.0) 261 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:40:35.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:40:35 smithi134 bash[24068]: audit 2024-07-15T06:40:34.182078+0000 mon.a (mon.0) 262 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:40:35.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:40:35 smithi134 bash[24068]: audit 2024-07-15T06:40:34.188949+0000 mon.a (mon.0) 263 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:40:35.374 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:40:35 smithi089 bash[21793]: audit 2024-07-15T06:40:34.180801+0000 mon.a (mon.0) 261 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:40:35.374 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:40:35 smithi089 bash[21793]: audit 2024-07-15T06:40:34.182078+0000 mon.a (mon.0) 262 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:40:35.375 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:40:35 smithi089 bash[21793]: audit 2024-07-15T06:40:34.188949+0000 mon.a (mon.0) 263 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:40:35.375 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:40:35 smithi089 bash[31372]: audit 2024-07-15T06:40:34.180801+0000 mon.a (mon.0) 261 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:40:35.375 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:40:35 smithi089 bash[31372]: audit 2024-07-15T06:40:34.182078+0000 mon.a (mon.0) 262 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:40:35.375 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:40:35 smithi089 bash[31372]: audit 2024-07-15T06:40:34.188949+0000 mon.a (mon.0) 263 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:40:36.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:40:36 smithi134 bash[24068]: cluster 2024-07-15T06:40:34.881424+0000 mgr.y (mgr.14150) 107 : cluster [DBG] pgmap v69: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:40:36.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:40:36 smithi134 bash[24068]: audit 2024-07-15T06:40:35.594676+0000 mon.a (mon.0) 264 : audit [INF] from='osd.0 [v2:172.21.15.89:6802/843843653,v1:172.21.15.89:6803/843843653]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-07-15T06:40:36.624 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:40:36 smithi089 bash[21793]: cluster 2024-07-15T06:40:34.881424+0000 mgr.y (mgr.14150) 107 : cluster [DBG] pgmap v69: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:40:36.624 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:40:36 smithi089 bash[21793]: audit 2024-07-15T06:40:35.594676+0000 mon.a (mon.0) 264 : audit [INF] from='osd.0 [v2:172.21.15.89:6802/843843653,v1:172.21.15.89:6803/843843653]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-07-15T06:40:36.624 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:40:36 smithi089 bash[31372]: cluster 2024-07-15T06:40:34.881424+0000 mgr.y (mgr.14150) 107 : cluster [DBG] pgmap v69: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:40:36.624 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:40:36 smithi089 bash[31372]: audit 2024-07-15T06:40:35.594676+0000 mon.a (mon.0) 264 : audit [INF] from='osd.0 [v2:172.21.15.89:6802/843843653,v1:172.21.15.89:6803/843843653]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-07-15T06:40:37.342 INFO:teuthology.orchestra.run.smithi089.stdout:Created osd(s) 0 on host 'smithi089' 2024-07-15T06:40:37.538 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:40:37 smithi089 bash[31372]: audit 2024-07-15T06:40:36.197713+0000 mon.a (mon.0) 265 : audit [INF] from='osd.0 [v2:172.21.15.89:6802/843843653,v1:172.21.15.89:6803/843843653]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-07-15T06:40:37.539 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:40:37 smithi089 bash[31372]: cluster 2024-07-15T06:40:36.197821+0000 mon.a (mon.0) 266 : cluster [DBG] osdmap e6: 1 total, 0 up, 1 in 2024-07-15T06:40:37.539 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:40:37 smithi089 bash[31372]: audit 2024-07-15T06:40:36.198434+0000 mon.a (mon.0) 267 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-07-15T06:40:37.539 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:40:37 smithi089 bash[31372]: audit 2024-07-15T06:40:36.198951+0000 mon.a (mon.0) 268 : audit [INF] from='osd.0 [v2:172.21.15.89:6802/843843653,v1:172.21.15.89:6803/843843653]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2024-07-15T06:40:37.539 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:40:37 smithi089 bash[31372]: audit 2024-07-15T06:40:37.197209+0000 mon.a (mon.0) 269 : audit [INF] from='osd.0 [v2:172.21.15.89:6802/843843653,v1:172.21.15.89:6803/843843653]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]': finished 2024-07-15T06:40:37.539 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:40:37 smithi089 bash[31372]: cluster 2024-07-15T06:40:37.197324+0000 mon.a (mon.0) 270 : cluster [DBG] osdmap e7: 1 total, 0 up, 1 in 2024-07-15T06:40:37.539 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:40:37 smithi089 bash[31372]: audit 2024-07-15T06:40:37.197567+0000 mon.a (mon.0) 271 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-07-15T06:40:37.539 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:40:37 smithi089 bash[21793]: audit 2024-07-15T06:40:36.197713+0000 mon.a (mon.0) 265 : audit [INF] from='osd.0 [v2:172.21.15.89:6802/843843653,v1:172.21.15.89:6803/843843653]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-07-15T06:40:37.539 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:40:37 smithi089 bash[21793]: cluster 2024-07-15T06:40:36.197821+0000 mon.a (mon.0) 266 : cluster [DBG] osdmap e6: 1 total, 0 up, 1 in 2024-07-15T06:40:37.539 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:40:37 smithi089 bash[21793]: audit 2024-07-15T06:40:36.198434+0000 mon.a (mon.0) 267 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-07-15T06:40:37.539 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:40:37 smithi089 bash[21793]: audit 2024-07-15T06:40:36.198951+0000 mon.a (mon.0) 268 : audit [INF] from='osd.0 [v2:172.21.15.89:6802/843843653,v1:172.21.15.89:6803/843843653]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2024-07-15T06:40:37.539 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:40:37 smithi089 bash[21793]: audit 2024-07-15T06:40:37.197209+0000 mon.a (mon.0) 269 : audit [INF] from='osd.0 [v2:172.21.15.89:6802/843843653,v1:172.21.15.89:6803/843843653]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]': finished 2024-07-15T06:40:37.539 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:40:37 smithi089 bash[21793]: cluster 2024-07-15T06:40:37.197324+0000 mon.a (mon.0) 270 : cluster [DBG] osdmap e7: 1 total, 0 up, 1 in 2024-07-15T06:40:37.540 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:40:37 smithi089 bash[21793]: audit 2024-07-15T06:40:37.197567+0000 mon.a (mon.0) 271 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-07-15T06:40:37.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:40:37 smithi134 bash[24068]: audit 2024-07-15T06:40:36.197713+0000 mon.a (mon.0) 265 : audit [INF] from='osd.0 [v2:172.21.15.89:6802/843843653,v1:172.21.15.89:6803/843843653]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-07-15T06:40:37.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:40:37 smithi134 bash[24068]: cluster 2024-07-15T06:40:36.197821+0000 mon.a (mon.0) 266 : cluster [DBG] osdmap e6: 1 total, 0 up, 1 in 2024-07-15T06:40:37.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:40:37 smithi134 bash[24068]: audit 2024-07-15T06:40:36.198434+0000 mon.a (mon.0) 267 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-07-15T06:40:37.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:40:37 smithi134 bash[24068]: audit 2024-07-15T06:40:36.198951+0000 mon.a (mon.0) 268 : audit [INF] from='osd.0 [v2:172.21.15.89:6802/843843653,v1:172.21.15.89:6803/843843653]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2024-07-15T06:40:37.560 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:40:37 smithi134 bash[24068]: audit 2024-07-15T06:40:37.197209+0000 mon.a (mon.0) 269 : audit [INF] from='osd.0 [v2:172.21.15.89:6802/843843653,v1:172.21.15.89:6803/843843653]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]': finished 2024-07-15T06:40:37.560 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:40:37 smithi134 bash[24068]: cluster 2024-07-15T06:40:37.197324+0000 mon.a (mon.0) 270 : cluster [DBG] osdmap e7: 1 total, 0 up, 1 in 2024-07-15T06:40:37.560 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:40:37 smithi134 bash[24068]: audit 2024-07-15T06:40:37.197567+0000 mon.a (mon.0) 271 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-07-15T06:40:37.969 DEBUG:teuthology.orchestra.run.smithi089:osd.0> sudo journalctl -f -n 0 -u ceph-75ba02bc-4274-11ef-bcad-c7b262605968@osd.0.service 2024-07-15T06:40:37.972 INFO:tasks.cephadm:Deploying osd.1 on smithi089 with /dev/vg_nvme/lv_3... 2024-07-15T06:40:37.972 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-07-15T06:40:38.375 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:40:38 smithi089 bash[21793]: cluster 2024-07-15T06:40:36.881829+0000 mgr.y (mgr.14150) 108 : cluster [DBG] pgmap v71: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:40:38.375 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:40:38 smithi089 bash[21793]: audit 2024-07-15T06:40:37.210104+0000 mon.a (mon.0) 272 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-07-15T06:40:38.375 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:40:38 smithi089 bash[21793]: audit 2024-07-15T06:40:37.325059+0000 mon.a (mon.0) 273 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:40:38.375 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:40:38 smithi089 bash[21793]: audit 2024-07-15T06:40:37.332725+0000 mon.a (mon.0) 274 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:40:38.375 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:40:38 smithi089 bash[21793]: audit 2024-07-15T06:40:37.340199+0000 mon.a (mon.0) 275 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:40:38.375 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:40:38 smithi089 bash[21793]: audit 2024-07-15T06:40:38.202448+0000 mon.a (mon.0) 276 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-07-15T06:40:38.375 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:40:38 smithi089 bash[31372]: cluster 2024-07-15T06:40:36.881829+0000 mgr.y (mgr.14150) 108 : cluster [DBG] pgmap v71: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:40:38.375 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:40:38 smithi089 bash[31372]: audit 2024-07-15T06:40:37.210104+0000 mon.a (mon.0) 272 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-07-15T06:40:38.376 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:40:38 smithi089 bash[31372]: audit 2024-07-15T06:40:37.325059+0000 mon.a (mon.0) 273 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:40:38.376 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:40:38 smithi089 bash[31372]: audit 2024-07-15T06:40:37.332725+0000 mon.a (mon.0) 274 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:40:38.376 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:40:38 smithi089 bash[31372]: audit 2024-07-15T06:40:37.340199+0000 mon.a (mon.0) 275 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:40:38.376 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:40:38 smithi089 bash[31372]: audit 2024-07-15T06:40:38.202448+0000 mon.a (mon.0) 276 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-07-15T06:40:38.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:40:38 smithi134 bash[24068]: cluster 2024-07-15T06:40:36.881829+0000 mgr.y (mgr.14150) 108 : cluster [DBG] pgmap v71: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:40:38.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:40:38 smithi134 bash[24068]: audit 2024-07-15T06:40:37.210104+0000 mon.a (mon.0) 272 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-07-15T06:40:38.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:40:38 smithi134 bash[24068]: audit 2024-07-15T06:40:37.325059+0000 mon.a (mon.0) 273 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:40:38.560 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:40:38 smithi134 bash[24068]: audit 2024-07-15T06:40:37.332725+0000 mon.a (mon.0) 274 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:40:38.560 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:40:38 smithi134 bash[24068]: audit 2024-07-15T06:40:37.340199+0000 mon.a (mon.0) 275 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:40:38.560 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:40:38 smithi134 bash[24068]: audit 2024-07-15T06:40:38.202448+0000 mon.a (mon.0) 276 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-07-15T06:40:39.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:40:39 smithi134 bash[24068]: cluster 2024-07-15T06:40:36.567594+0000 osd.0 (osd.0) 1 : cluster [DBG] purged_snaps scrub starts 2024-07-15T06:40:39.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:40:39 smithi134 bash[24068]: cluster 2024-07-15T06:40:36.567650+0000 osd.0 (osd.0) 2 : cluster [DBG] purged_snaps scrub ok 2024-07-15T06:40:39.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:40:39 smithi134 bash[24068]: cluster 2024-07-15T06:40:38.214780+0000 mon.a (mon.0) 277 : cluster [INF] osd.0 [v2:172.21.15.89:6802/843843653,v1:172.21.15.89:6803/843843653] boot 2024-07-15T06:40:39.560 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:40:39 smithi134 bash[24068]: cluster 2024-07-15T06:40:38.214805+0000 mon.a (mon.0) 278 : cluster [DBG] osdmap e8: 1 total, 1 up, 1 in 2024-07-15T06:40:39.560 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:40:39 smithi134 bash[24068]: audit 2024-07-15T06:40:38.215019+0000 mon.a (mon.0) 279 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-07-15T06:40:39.624 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:40:39 smithi089 bash[21793]: cluster 2024-07-15T06:40:36.567594+0000 osd.0 (osd.0) 1 : cluster [DBG] purged_snaps scrub starts 2024-07-15T06:40:39.624 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:40:39 smithi089 bash[21793]: cluster 2024-07-15T06:40:36.567650+0000 osd.0 (osd.0) 2 : cluster [DBG] purged_snaps scrub ok 2024-07-15T06:40:39.625 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:40:39 smithi089 bash[21793]: cluster 2024-07-15T06:40:38.214780+0000 mon.a (mon.0) 277 : cluster [INF] osd.0 [v2:172.21.15.89:6802/843843653,v1:172.21.15.89:6803/843843653] boot 2024-07-15T06:40:39.625 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:40:39 smithi089 bash[21793]: cluster 2024-07-15T06:40:38.214805+0000 mon.a (mon.0) 278 : cluster [DBG] osdmap e8: 1 total, 1 up, 1 in 2024-07-15T06:40:39.625 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:40:39 smithi089 bash[21793]: audit 2024-07-15T06:40:38.215019+0000 mon.a (mon.0) 279 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-07-15T06:40:39.625 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:40:39 smithi089 bash[31372]: cluster 2024-07-15T06:40:36.567594+0000 osd.0 (osd.0) 1 : cluster [DBG] purged_snaps scrub starts 2024-07-15T06:40:39.625 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:40:39 smithi089 bash[31372]: cluster 2024-07-15T06:40:36.567650+0000 osd.0 (osd.0) 2 : cluster [DBG] purged_snaps scrub ok 2024-07-15T06:40:39.625 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:40:39 smithi089 bash[31372]: cluster 2024-07-15T06:40:38.214780+0000 mon.a (mon.0) 277 : cluster [INF] osd.0 [v2:172.21.15.89:6802/843843653,v1:172.21.15.89:6803/843843653] boot 2024-07-15T06:40:39.625 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:40:39 smithi089 bash[31372]: cluster 2024-07-15T06:40:38.214805+0000 mon.a (mon.0) 278 : cluster [DBG] osdmap e8: 1 total, 1 up, 1 in 2024-07-15T06:40:39.625 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:40:39 smithi089 bash[31372]: audit 2024-07-15T06:40:38.215019+0000 mon.a (mon.0) 279 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-07-15T06:40:40.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:40:40 smithi134 bash[24068]: cluster 2024-07-15T06:40:38.882236+0000 mgr.y (mgr.14150) 109 : cluster [DBG] pgmap v74: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:40:40.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:40:40 smithi134 bash[24068]: cluster 2024-07-15T06:40:39.226464+0000 mon.a (mon.0) 280 : cluster [DBG] osdmap e9: 1 total, 1 up, 1 in 2024-07-15T06:40:40.624 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:40:40 smithi089 bash[21793]: cluster 2024-07-15T06:40:38.882236+0000 mgr.y (mgr.14150) 109 : cluster [DBG] pgmap v74: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:40:40.624 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:40:40 smithi089 bash[21793]: cluster 2024-07-15T06:40:39.226464+0000 mon.a (mon.0) 280 : cluster [DBG] osdmap e9: 1 total, 1 up, 1 in 2024-07-15T06:40:40.625 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:40:40 smithi089 bash[31372]: cluster 2024-07-15T06:40:38.882236+0000 mgr.y (mgr.14150) 109 : cluster [DBG] pgmap v74: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-15T06:40:40.625 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:40:40 smithi089 bash[31372]: cluster 2024-07-15T06:40:39.226464+0000 mon.a (mon.0) 280 : cluster [DBG] osdmap e9: 1 total, 1 up, 1 in 2024-07-15T06:40:42.088 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T06:40:42.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:40:42 smithi134 bash[24068]: cluster 2024-07-15T06:40:40.882726+0000 mgr.y (mgr.14150) 110 : cluster [DBG] pgmap v76: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-15T06:40:42.624 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:40:42 smithi089 bash[21793]: cluster 2024-07-15T06:40:40.882726+0000 mgr.y (mgr.14150) 110 : cluster [DBG] pgmap v76: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-15T06:40:42.625 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:40:42 smithi089 bash[31372]: cluster 2024-07-15T06:40:40.882726+0000 mgr.y (mgr.14150) 110 : cluster [DBG] pgmap v76: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-15T06:40:44.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:40:44 smithi134 bash[24068]: cluster 2024-07-15T06:40:42.883183+0000 mgr.y (mgr.14150) 111 : cluster [DBG] pgmap v77: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-15T06:40:44.624 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:40:44 smithi089 bash[21793]: cluster 2024-07-15T06:40:42.883183+0000 mgr.y (mgr.14150) 111 : cluster [DBG] pgmap v77: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-15T06:40:44.624 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:40:44 smithi089 bash[31372]: cluster 2024-07-15T06:40:42.883183+0000 mgr.y (mgr.14150) 111 : cluster [DBG] pgmap v77: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-15T06:40:46.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:40:46 smithi134 bash[24068]: cluster 2024-07-15T06:40:44.883606+0000 mgr.y (mgr.14150) 112 : cluster [DBG] pgmap v78: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-15T06:40:46.624 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:40:46 smithi089 bash[21793]: cluster 2024-07-15T06:40:44.883606+0000 mgr.y (mgr.14150) 112 : cluster [DBG] pgmap v78: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-15T06:40:46.624 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:40:46 smithi089 bash[31372]: cluster 2024-07-15T06:40:44.883606+0000 mgr.y (mgr.14150) 112 : cluster [DBG] pgmap v78: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-15T06:40:46.722 INFO:teuthology.orchestra.run.smithi089.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-07-15T06:40:46.890 INFO:teuthology.orchestra.run.smithi089.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-07-15T06:40:46.906 INFO:teuthology.orchestra.run.smithi089.stderr: stderr: 10+0 records in 2024-07-15T06:40:46.907 INFO:teuthology.orchestra.run.smithi089.stderr:10+0 records out 2024-07-15T06:40:46.907 INFO:teuthology.orchestra.run.smithi089.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0145222 s, 722 MB/s 2024-07-15T06:40:46.907 INFO:teuthology.orchestra.run.smithi089.stderr:--> Zapping successful for: 2024-07-15T06:40:47.637 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph orch daemon add osd smithi089:vg_nvme/lv_3 2024-07-15T06:40:48.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:40:48 smithi134 bash[24068]: cluster 2024-07-15T06:40:46.883904+0000 mgr.y (mgr.14150) 113 : cluster [DBG] pgmap v79: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-15T06:40:48.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:40:48 smithi134 bash[24068]: audit 2024-07-15T06:40:48.013401+0000 mon.a (mon.0) 281 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:40:48.560 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:40:48 smithi134 bash[24068]: audit 2024-07-15T06:40:48.021488+0000 mon.a (mon.0) 282 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:40:48.560 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:40:48 smithi134 bash[24068]: audit 2024-07-15T06:40:48.023214+0000 mon.a (mon.0) 283 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:40:48.560 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:40:48 smithi134 bash[24068]: audit 2024-07-15T06:40:48.032782+0000 mon.a (mon.0) 284 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:40:48.560 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:40:48 smithi134 bash[24068]: audit 2024-07-15T06:40:48.034806+0000 mon.a (mon.0) 285 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:40:48.560 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:40:48 smithi134 bash[24068]: audit 2024-07-15T06:40:48.036362+0000 mon.a (mon.0) 286 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:40:48.560 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:40:48 smithi134 bash[24068]: audit 2024-07-15T06:40:48.046046+0000 mon.a (mon.0) 287 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:40:48.625 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:40:48 smithi089 bash[21793]: cluster 2024-07-15T06:40:46.883904+0000 mgr.y (mgr.14150) 113 : cluster [DBG] pgmap v79: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-15T06:40:48.625 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:40:48 smithi089 bash[21793]: audit 2024-07-15T06:40:48.013401+0000 mon.a (mon.0) 281 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:40:48.625 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:40:48 smithi089 bash[21793]: audit 2024-07-15T06:40:48.021488+0000 mon.a (mon.0) 282 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:40:48.625 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:40:48 smithi089 bash[21793]: audit 2024-07-15T06:40:48.023214+0000 mon.a (mon.0) 283 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:40:48.625 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:40:48 smithi089 bash[21793]: audit 2024-07-15T06:40:48.032782+0000 mon.a (mon.0) 284 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:40:48.625 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:40:48 smithi089 bash[21793]: audit 2024-07-15T06:40:48.034806+0000 mon.a (mon.0) 285 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:40:48.625 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:40:48 smithi089 bash[21793]: audit 2024-07-15T06:40:48.036362+0000 mon.a (mon.0) 286 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:40:48.625 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:40:48 smithi089 bash[21793]: audit 2024-07-15T06:40:48.046046+0000 mon.a (mon.0) 287 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:40:48.626 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:40:48 smithi089 bash[31372]: cluster 2024-07-15T06:40:46.883904+0000 mgr.y (mgr.14150) 113 : cluster [DBG] pgmap v79: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-15T06:40:48.626 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:40:48 smithi089 bash[31372]: audit 2024-07-15T06:40:48.013401+0000 mon.a (mon.0) 281 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:40:48.626 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:40:48 smithi089 bash[31372]: audit 2024-07-15T06:40:48.021488+0000 mon.a (mon.0) 282 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:40:48.626 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:40:48 smithi089 bash[31372]: audit 2024-07-15T06:40:48.023214+0000 mon.a (mon.0) 283 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:40:48.626 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:40:48 smithi089 bash[31372]: audit 2024-07-15T06:40:48.032782+0000 mon.a (mon.0) 284 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:40:48.626 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:40:48 smithi089 bash[31372]: audit 2024-07-15T06:40:48.034806+0000 mon.a (mon.0) 285 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:40:48.626 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:40:48 smithi089 bash[31372]: audit 2024-07-15T06:40:48.036362+0000 mon.a (mon.0) 286 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:40:48.626 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:40:48 smithi089 bash[31372]: audit 2024-07-15T06:40:48.046046+0000 mon.a (mon.0) 287 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:40:49.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:40:49 smithi134 bash[24068]: cephadm 2024-07-15T06:40:48.004705+0000 mgr.y (mgr.14150) 114 : cephadm [INF] Detected new or changed devices on smithi089 2024-07-15T06:40:49.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:40:49 smithi134 bash[24068]: cephadm 2024-07-15T06:40:48.024417+0000 mgr.y (mgr.14150) 115 : cephadm [INF] Adjusting osd_memory_target on smithi089 to 16241M 2024-07-15T06:40:49.624 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:40:49 smithi089 bash[21793]: cephadm 2024-07-15T06:40:48.004705+0000 mgr.y (mgr.14150) 114 : cephadm [INF] Detected new or changed devices on smithi089 2024-07-15T06:40:49.624 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:40:49 smithi089 bash[21793]: cephadm 2024-07-15T06:40:48.024417+0000 mgr.y (mgr.14150) 115 : cephadm [INF] Adjusting osd_memory_target on smithi089 to 16241M 2024-07-15T06:40:49.625 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:40:49 smithi089 bash[31372]: cephadm 2024-07-15T06:40:48.004705+0000 mgr.y (mgr.14150) 114 : cephadm [INF] Detected new or changed devices on smithi089 2024-07-15T06:40:49.625 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:40:49 smithi089 bash[31372]: cephadm 2024-07-15T06:40:48.024417+0000 mgr.y (mgr.14150) 115 : cephadm [INF] Adjusting osd_memory_target on smithi089 to 16241M 2024-07-15T06:40:50.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:40:50 smithi134 bash[24068]: cluster 2024-07-15T06:40:48.884348+0000 mgr.y (mgr.14150) 116 : cluster [DBG] pgmap v80: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-15T06:40:50.624 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:40:50 smithi089 bash[21793]: cluster 2024-07-15T06:40:48.884348+0000 mgr.y (mgr.14150) 116 : cluster [DBG] pgmap v80: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-15T06:40:50.625 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:40:50 smithi089 bash[31372]: cluster 2024-07-15T06:40:48.884348+0000 mgr.y (mgr.14150) 116 : cluster [DBG] pgmap v80: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-15T06:40:52.368 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T06:40:52.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:40:52 smithi134 bash[24068]: cluster 2024-07-15T06:40:50.884813+0000 mgr.y (mgr.14150) 117 : cluster [DBG] pgmap v81: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-15T06:40:52.624 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:40:52 smithi089 bash[21793]: cluster 2024-07-15T06:40:50.884813+0000 mgr.y (mgr.14150) 117 : cluster [DBG] pgmap v81: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-15T06:40:52.624 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:40:52 smithi089 bash[31372]: cluster 2024-07-15T06:40:50.884813+0000 mgr.y (mgr.14150) 117 : cluster [DBG] pgmap v81: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-15T06:40:54.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:40:54 smithi134 bash[24068]: cluster 2024-07-15T06:40:52.885333+0000 mgr.y (mgr.14150) 118 : cluster [DBG] pgmap v82: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-15T06:40:54.624 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:40:54 smithi089 bash[21793]: cluster 2024-07-15T06:40:52.885333+0000 mgr.y (mgr.14150) 118 : cluster [DBG] pgmap v82: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-15T06:40:54.624 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:40:54 smithi089 bash[31372]: cluster 2024-07-15T06:40:52.885333+0000 mgr.y (mgr.14150) 118 : cluster [DBG] pgmap v82: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-15T06:40:56.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:40:56 smithi134 bash[24068]: cluster 2024-07-15T06:40:54.885888+0000 mgr.y (mgr.14150) 119 : cluster [DBG] pgmap v83: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-15T06:40:56.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:40:56 smithi134 bash[24068]: audit 2024-07-15T06:40:55.350097+0000 mgr.y (mgr.14150) 120 : audit [DBG] from='client.14235 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi089:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T06:40:56.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:40:56 smithi134 bash[24068]: audit 2024-07-15T06:40:55.356300+0000 mon.a (mon.0) 288 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-07-15T06:40:56.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:40:56 smithi134 bash[24068]: audit 2024-07-15T06:40:55.361717+0000 mon.a (mon.0) 289 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-07-15T06:40:56.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:40:56 smithi134 bash[24068]: audit 2024-07-15T06:40:55.363100+0000 mon.a (mon.0) 290 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:40:56.624 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:40:56 smithi089 bash[21793]: cluster 2024-07-15T06:40:54.885888+0000 mgr.y (mgr.14150) 119 : cluster [DBG] pgmap v83: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-15T06:40:56.624 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:40:56 smithi089 bash[21793]: audit 2024-07-15T06:40:55.350097+0000 mgr.y (mgr.14150) 120 : audit [DBG] from='client.14235 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi089:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T06:40:56.624 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:40:56 smithi089 bash[21793]: audit 2024-07-15T06:40:55.356300+0000 mon.a (mon.0) 288 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-07-15T06:40:56.624 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:40:56 smithi089 bash[21793]: audit 2024-07-15T06:40:55.361717+0000 mon.a (mon.0) 289 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-07-15T06:40:56.625 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:40:56 smithi089 bash[21793]: audit 2024-07-15T06:40:55.363100+0000 mon.a (mon.0) 290 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:40:56.625 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:40:56 smithi089 bash[31372]: cluster 2024-07-15T06:40:54.885888+0000 mgr.y (mgr.14150) 119 : cluster [DBG] pgmap v83: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-15T06:40:56.625 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:40:56 smithi089 bash[31372]: audit 2024-07-15T06:40:55.350097+0000 mgr.y (mgr.14150) 120 : audit [DBG] from='client.14235 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi089:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T06:40:56.625 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:40:56 smithi089 bash[31372]: audit 2024-07-15T06:40:55.356300+0000 mon.a (mon.0) 288 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-07-15T06:40:56.625 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:40:56 smithi089 bash[31372]: audit 2024-07-15T06:40:55.361717+0000 mon.a (mon.0) 289 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-07-15T06:40:56.625 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:40:56 smithi089 bash[31372]: audit 2024-07-15T06:40:55.363100+0000 mon.a (mon.0) 290 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:40:58.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:40:58 smithi134 bash[24068]: cluster 2024-07-15T06:40:56.886423+0000 mgr.y (mgr.14150) 121 : cluster [DBG] pgmap v84: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-15T06:40:58.624 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:40:58 smithi089 bash[21793]: cluster 2024-07-15T06:40:56.886423+0000 mgr.y (mgr.14150) 121 : cluster [DBG] pgmap v84: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-15T06:40:58.624 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:40:58 smithi089 bash[31372]: cluster 2024-07-15T06:40:56.886423+0000 mgr.y (mgr.14150) 121 : cluster [DBG] pgmap v84: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-15T06:41:00.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:41:00 smithi134 bash[24068]: cluster 2024-07-15T06:40:58.886877+0000 mgr.y (mgr.14150) 122 : cluster [DBG] pgmap v85: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-15T06:41:00.624 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:41:00 smithi089 bash[21793]: cluster 2024-07-15T06:40:58.886877+0000 mgr.y (mgr.14150) 122 : cluster [DBG] pgmap v85: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-15T06:41:00.624 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:41:00 smithi089 bash[31372]: cluster 2024-07-15T06:40:58.886877+0000 mgr.y (mgr.14150) 122 : cluster [DBG] pgmap v85: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-15T06:41:02.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:41:02 smithi134 bash[24068]: cluster 2024-07-15T06:41:00.887430+0000 mgr.y (mgr.14150) 123 : cluster [DBG] pgmap v86: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-15T06:41:02.572 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:41:02 smithi089 bash[21793]: cluster 2024-07-15T06:41:00.887430+0000 mgr.y (mgr.14150) 123 : cluster [DBG] pgmap v86: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-15T06:41:02.572 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:41:02 smithi089 bash[31372]: cluster 2024-07-15T06:41:00.887430+0000 mgr.y (mgr.14150) 123 : cluster [DBG] pgmap v86: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-15T06:41:04.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:41:04 smithi134 bash[24068]: cluster 2024-07-15T06:41:02.887916+0000 mgr.y (mgr.14150) 124 : cluster [DBG] pgmap v87: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-15T06:41:04.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:41:04 smithi134 bash[24068]: audit 2024-07-15T06:41:03.314180+0000 mon.c (mon.2) 4 : audit [INF] from='client.? 172.21.15.89:0/3050075501' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d304859e-317a-414e-a16c-f097fc4b2702"}]: dispatch 2024-07-15T06:41:04.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:41:04 smithi134 bash[24068]: audit 2024-07-15T06:41:03.314782+0000 mon.a (mon.0) 291 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d304859e-317a-414e-a16c-f097fc4b2702"}]: dispatch 2024-07-15T06:41:04.560 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:41:04 smithi134 bash[24068]: audit 2024-07-15T06:41:03.319499+0000 mon.a (mon.0) 292 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d304859e-317a-414e-a16c-f097fc4b2702"}]': finished 2024-07-15T06:41:04.560 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:41:04 smithi134 bash[24068]: cluster 2024-07-15T06:41:03.319537+0000 mon.a (mon.0) 293 : cluster [DBG] osdmap e10: 2 total, 1 up, 2 in 2024-07-15T06:41:04.560 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:41:04 smithi134 bash[24068]: audit 2024-07-15T06:41:03.319639+0000 mon.a (mon.0) 294 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-07-15T06:41:04.560 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:41:04 smithi134 bash[24068]: audit 2024-07-15T06:41:03.940722+0000 mon.a (mon.0) 295 : audit [DBG] from='client.? 172.21.15.89:0/3056496922' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-07-15T06:41:04.624 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:41:04 smithi089 bash[21793]: cluster 2024-07-15T06:41:02.887916+0000 mgr.y (mgr.14150) 124 : cluster [DBG] pgmap v87: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-15T06:41:04.624 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:41:04 smithi089 bash[21793]: audit 2024-07-15T06:41:03.314180+0000 mon.c (mon.2) 4 : audit [INF] from='client.? 172.21.15.89:0/3050075501' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d304859e-317a-414e-a16c-f097fc4b2702"}]: dispatch 2024-07-15T06:41:04.624 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:41:04 smithi089 bash[21793]: audit 2024-07-15T06:41:03.314782+0000 mon.a (mon.0) 291 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d304859e-317a-414e-a16c-f097fc4b2702"}]: dispatch 2024-07-15T06:41:04.624 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:41:04 smithi089 bash[21793]: audit 2024-07-15T06:41:03.319499+0000 mon.a (mon.0) 292 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d304859e-317a-414e-a16c-f097fc4b2702"}]': finished 2024-07-15T06:41:04.624 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:41:04 smithi089 bash[21793]: cluster 2024-07-15T06:41:03.319537+0000 mon.a (mon.0) 293 : cluster [DBG] osdmap e10: 2 total, 1 up, 2 in 2024-07-15T06:41:04.625 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:41:04 smithi089 bash[21793]: audit 2024-07-15T06:41:03.319639+0000 mon.a (mon.0) 294 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-07-15T06:41:04.625 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:41:04 smithi089 bash[21793]: audit 2024-07-15T06:41:03.940722+0000 mon.a (mon.0) 295 : audit [DBG] from='client.? 172.21.15.89:0/3056496922' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-07-15T06:41:04.625 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:41:04 smithi089 bash[31372]: cluster 2024-07-15T06:41:02.887916+0000 mgr.y (mgr.14150) 124 : cluster [DBG] pgmap v87: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-15T06:41:04.625 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:41:04 smithi089 bash[31372]: audit 2024-07-15T06:41:03.314180+0000 mon.c (mon.2) 4 : audit [INF] from='client.? 172.21.15.89:0/3050075501' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d304859e-317a-414e-a16c-f097fc4b2702"}]: dispatch 2024-07-15T06:41:04.625 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:41:04 smithi089 bash[31372]: audit 2024-07-15T06:41:03.314782+0000 mon.a (mon.0) 291 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d304859e-317a-414e-a16c-f097fc4b2702"}]: dispatch 2024-07-15T06:41:04.625 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:41:04 smithi089 bash[31372]: audit 2024-07-15T06:41:03.319499+0000 mon.a (mon.0) 292 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d304859e-317a-414e-a16c-f097fc4b2702"}]': finished 2024-07-15T06:41:04.625 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:41:04 smithi089 bash[31372]: cluster 2024-07-15T06:41:03.319537+0000 mon.a (mon.0) 293 : cluster [DBG] osdmap e10: 2 total, 1 up, 2 in 2024-07-15T06:41:04.625 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:41:04 smithi089 bash[31372]: audit 2024-07-15T06:41:03.319639+0000 mon.a (mon.0) 294 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-07-15T06:41:04.625 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:41:04 smithi089 bash[31372]: audit 2024-07-15T06:41:03.940722+0000 mon.a (mon.0) 295 : audit [DBG] from='client.? 172.21.15.89:0/3056496922' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-07-15T06:41:06.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:41:06 smithi134 bash[24068]: cluster 2024-07-15T06:41:04.888247+0000 mgr.y (mgr.14150) 125 : cluster [DBG] pgmap v89: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-15T06:41:06.569 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:41:06 smithi089 bash[21793]: cluster 2024-07-15T06:41:04.888247+0000 mgr.y (mgr.14150) 125 : cluster [DBG] pgmap v89: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-15T06:41:06.570 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:41:06 smithi089 bash[31372]: cluster 2024-07-15T06:41:04.888247+0000 mgr.y (mgr.14150) 125 : cluster [DBG] pgmap v89: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-15T06:41:08.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:41:08 smithi134 bash[24068]: cluster 2024-07-15T06:41:06.888644+0000 mgr.y (mgr.14150) 126 : cluster [DBG] pgmap v90: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-15T06:41:08.624 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:41:08 smithi089 bash[21793]: cluster 2024-07-15T06:41:06.888644+0000 mgr.y (mgr.14150) 126 : cluster [DBG] pgmap v90: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-15T06:41:08.624 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:41:08 smithi089 bash[31372]: cluster 2024-07-15T06:41:06.888644+0000 mgr.y (mgr.14150) 126 : cluster [DBG] pgmap v90: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-15T06:41:10.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:41:10 smithi134 bash[24068]: cluster 2024-07-15T06:41:08.889076+0000 mgr.y (mgr.14150) 127 : cluster [DBG] pgmap v91: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-15T06:41:10.624 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:41:10 smithi089 bash[21793]: cluster 2024-07-15T06:41:08.889076+0000 mgr.y (mgr.14150) 127 : cluster [DBG] pgmap v91: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-15T06:41:10.624 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:41:10 smithi089 bash[31372]: cluster 2024-07-15T06:41:08.889076+0000 mgr.y (mgr.14150) 127 : cluster [DBG] pgmap v91: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-15T06:41:12.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:41:12 smithi134 bash[24068]: cluster 2024-07-15T06:41:10.889509+0000 mgr.y (mgr.14150) 128 : cluster [DBG] pgmap v92: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-15T06:41:12.624 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:41:12 smithi089 bash[21793]: cluster 2024-07-15T06:41:10.889509+0000 mgr.y (mgr.14150) 128 : cluster [DBG] pgmap v92: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-15T06:41:12.624 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:41:12 smithi089 bash[31372]: cluster 2024-07-15T06:41:10.889509+0000 mgr.y (mgr.14150) 128 : cluster [DBG] pgmap v92: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-15T06:41:14.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:41:14 smithi134 bash[24068]: cluster 2024-07-15T06:41:12.889835+0000 mgr.y (mgr.14150) 129 : cluster [DBG] pgmap v93: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-15T06:41:14.624 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:41:14 smithi089 bash[21793]: cluster 2024-07-15T06:41:12.889835+0000 mgr.y (mgr.14150) 129 : cluster [DBG] pgmap v93: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-15T06:41:14.624 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:41:14 smithi089 bash[31372]: cluster 2024-07-15T06:41:12.889835+0000 mgr.y (mgr.14150) 129 : cluster [DBG] pgmap v93: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-15T06:41:15.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:41:15 smithi134 bash[24068]: audit 2024-07-15T06:41:15.247446+0000 mon.a (mon.0) 296 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-07-15T06:41:15.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:41:15 smithi134 bash[24068]: audit 2024-07-15T06:41:15.248793+0000 mon.a (mon.0) 297 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:41:15.592 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:41:15 smithi089 bash[21793]: audit 2024-07-15T06:41:15.247446+0000 mon.a (mon.0) 296 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-07-15T06:41:15.593 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:41:15 smithi089 bash[21793]: audit 2024-07-15T06:41:15.248793+0000 mon.a (mon.0) 297 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:41:15.593 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:41:15 smithi089 bash[31372]: audit 2024-07-15T06:41:15.247446+0000 mon.a (mon.0) 296 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-07-15T06:41:15.593 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:41:15 smithi089 bash[31372]: audit 2024-07-15T06:41:15.248793+0000 mon.a (mon.0) 297 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:41:16.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:41:16 smithi134 bash[24068]: cluster 2024-07-15T06:41:14.890307+0000 mgr.y (mgr.14150) 130 : cluster [DBG] pgmap v94: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-15T06:41:16.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:41:16 smithi134 bash[24068]: cephadm 2024-07-15T06:41:15.250006+0000 mgr.y (mgr.14150) 131 : cephadm [INF] Deploying daemon osd.1 on smithi089 2024-07-15T06:41:16.624 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:41:16 smithi089 bash[21793]: cluster 2024-07-15T06:41:14.890307+0000 mgr.y (mgr.14150) 130 : cluster [DBG] pgmap v94: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-15T06:41:16.624 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:41:16 smithi089 bash[21793]: cephadm 2024-07-15T06:41:15.250006+0000 mgr.y (mgr.14150) 131 : cephadm [INF] Deploying daemon osd.1 on smithi089 2024-07-15T06:41:16.624 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:41:16 smithi089 bash[31372]: cluster 2024-07-15T06:41:14.890307+0000 mgr.y (mgr.14150) 130 : cluster [DBG] pgmap v94: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-15T06:41:16.624 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:41:16 smithi089 bash[31372]: cephadm 2024-07-15T06:41:15.250006+0000 mgr.y (mgr.14150) 131 : cephadm [INF] Deploying daemon osd.1 on smithi089 2024-07-15T06:41:18.624 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:41:18 smithi089 bash[31372]: cluster 2024-07-15T06:41:16.890858+0000 mgr.y (mgr.14150) 132 : cluster [DBG] pgmap v95: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-15T06:41:18.624 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:41:18 smithi089 bash[21793]: cluster 2024-07-15T06:41:16.890858+0000 mgr.y (mgr.14150) 132 : cluster [DBG] pgmap v95: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-15T06:41:18.809 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:41:18 smithi134 bash[24068]: cluster 2024-07-15T06:41:16.890858+0000 mgr.y (mgr.14150) 132 : cluster [DBG] pgmap v95: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-15T06:41:19.170 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:41:19 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:41:19.171 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:41:19 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:41:19.171 INFO:journalctl@ceph.osd.0.smithi089.stdout:Jul 15 06:41:19 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:41:19.171 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:41:19 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:41:19.526 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:41:19 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:41:19.526 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:41:19 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:41:19.527 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:41:19 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:41:19.527 INFO:journalctl@ceph.osd.0.smithi089.stdout:Jul 15 06:41:19 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:41:20.624 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:41:20 smithi089 bash[21793]: cluster 2024-07-15T06:41:18.891227+0000 mgr.y (mgr.14150) 133 : cluster [DBG] pgmap v96: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-15T06:41:20.624 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:41:20 smithi089 bash[21793]: audit 2024-07-15T06:41:19.443298+0000 mon.a (mon.0) 298 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:41:20.624 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:41:20 smithi089 bash[21793]: audit 2024-07-15T06:41:19.448039+0000 mon.a (mon.0) 299 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:41:20.624 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:41:20 smithi089 bash[21793]: audit 2024-07-15T06:41:19.452408+0000 mon.a (mon.0) 300 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:41:20.624 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:41:20 smithi089 bash[31372]: cluster 2024-07-15T06:41:18.891227+0000 mgr.y (mgr.14150) 133 : cluster [DBG] pgmap v96: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-15T06:41:20.624 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:41:20 smithi089 bash[31372]: audit 2024-07-15T06:41:19.443298+0000 mon.a (mon.0) 298 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:41:20.625 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:41:20 smithi089 bash[31372]: audit 2024-07-15T06:41:19.448039+0000 mon.a (mon.0) 299 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:41:20.625 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:41:20 smithi089 bash[31372]: audit 2024-07-15T06:41:19.452408+0000 mon.a (mon.0) 300 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:41:20.809 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:41:20 smithi134 bash[24068]: cluster 2024-07-15T06:41:18.891227+0000 mgr.y (mgr.14150) 133 : cluster [DBG] pgmap v96: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-15T06:41:20.809 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:41:20 smithi134 bash[24068]: audit 2024-07-15T06:41:19.443298+0000 mon.a (mon.0) 298 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:41:20.809 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:41:20 smithi134 bash[24068]: audit 2024-07-15T06:41:19.448039+0000 mon.a (mon.0) 299 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:41:20.809 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:41:20 smithi134 bash[24068]: audit 2024-07-15T06:41:19.452408+0000 mon.a (mon.0) 300 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:41:22.624 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:41:22 smithi089 bash[21793]: cluster 2024-07-15T06:41:20.891487+0000 mgr.y (mgr.14150) 134 : cluster [DBG] pgmap v97: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-15T06:41:22.624 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:41:22 smithi089 bash[31372]: cluster 2024-07-15T06:41:20.891487+0000 mgr.y (mgr.14150) 134 : cluster [DBG] pgmap v97: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-15T06:41:22.809 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:41:22 smithi134 bash[24068]: cluster 2024-07-15T06:41:20.891487+0000 mgr.y (mgr.14150) 134 : cluster [DBG] pgmap v97: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-15T06:41:24.624 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:41:24 smithi089 bash[21793]: cluster 2024-07-15T06:41:22.891767+0000 mgr.y (mgr.14150) 135 : cluster [DBG] pgmap v98: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-15T06:41:24.624 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:41:24 smithi089 bash[31372]: cluster 2024-07-15T06:41:22.891767+0000 mgr.y (mgr.14150) 135 : cluster [DBG] pgmap v98: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-15T06:41:24.809 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:41:24 smithi134 bash[24068]: cluster 2024-07-15T06:41:22.891767+0000 mgr.y (mgr.14150) 135 : cluster [DBG] pgmap v98: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-15T06:41:25.624 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:41:25 smithi089 bash[21793]: audit 2024-07-15T06:41:24.477550+0000 mon.a (mon.0) 301 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:41:25.624 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:41:25 smithi089 bash[21793]: audit 2024-07-15T06:41:24.481511+0000 mon.a (mon.0) 302 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:41:25.624 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:41:25 smithi089 bash[21793]: audit 2024-07-15T06:41:24.482311+0000 mon.a (mon.0) 303 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:41:25.624 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:41:25 smithi089 bash[21793]: audit 2024-07-15T06:41:24.483025+0000 mon.a (mon.0) 304 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:41:25.624 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:41:25 smithi089 bash[21793]: audit 2024-07-15T06:41:24.487229+0000 mon.a (mon.0) 305 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:41:25.625 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:41:25 smithi089 bash[21793]: cluster 2024-07-15T06:41:24.892176+0000 mgr.y (mgr.14150) 136 : cluster [DBG] pgmap v99: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-15T06:41:25.625 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:41:25 smithi089 bash[31372]: audit 2024-07-15T06:41:24.477550+0000 mon.a (mon.0) 301 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:41:25.625 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:41:25 smithi089 bash[31372]: audit 2024-07-15T06:41:24.481511+0000 mon.a (mon.0) 302 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:41:25.625 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:41:25 smithi089 bash[31372]: audit 2024-07-15T06:41:24.482311+0000 mon.a (mon.0) 303 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:41:25.625 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:41:25 smithi089 bash[31372]: audit 2024-07-15T06:41:24.483025+0000 mon.a (mon.0) 304 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:41:25.625 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:41:25 smithi089 bash[31372]: audit 2024-07-15T06:41:24.487229+0000 mon.a (mon.0) 305 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:41:25.625 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:41:25 smithi089 bash[31372]: cluster 2024-07-15T06:41:24.892176+0000 mgr.y (mgr.14150) 136 : cluster [DBG] pgmap v99: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-15T06:41:25.809 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:41:25 smithi134 bash[24068]: audit 2024-07-15T06:41:24.477550+0000 mon.a (mon.0) 301 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:41:25.809 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:41:25 smithi134 bash[24068]: audit 2024-07-15T06:41:24.481511+0000 mon.a (mon.0) 302 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:41:25.809 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:41:25 smithi134 bash[24068]: audit 2024-07-15T06:41:24.482311+0000 mon.a (mon.0) 303 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:41:25.809 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:41:25 smithi134 bash[24068]: audit 2024-07-15T06:41:24.483025+0000 mon.a (mon.0) 304 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:41:25.809 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:41:25 smithi134 bash[24068]: audit 2024-07-15T06:41:24.487229+0000 mon.a (mon.0) 305 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:41:25.809 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:41:25 smithi134 bash[24068]: cluster 2024-07-15T06:41:24.892176+0000 mgr.y (mgr.14150) 136 : cluster [DBG] pgmap v99: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-15T06:41:26.624 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:41:26 smithi089 bash[21793]: audit 2024-07-15T06:41:25.852788+0000 mon.c (mon.2) 5 : audit [INF] from='osd.1 [v2:172.21.15.89:6810/1536075263,v1:172.21.15.89:6811/1536075263]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-07-15T06:41:26.624 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:41:26 smithi089 bash[21793]: audit 2024-07-15T06:41:25.853133+0000 mon.a (mon.0) 306 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-07-15T06:41:26.624 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:41:26 smithi089 bash[31372]: audit 2024-07-15T06:41:25.852788+0000 mon.c (mon.2) 5 : audit [INF] from='osd.1 [v2:172.21.15.89:6810/1536075263,v1:172.21.15.89:6811/1536075263]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-07-15T06:41:26.624 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:41:26 smithi089 bash[31372]: audit 2024-07-15T06:41:25.853133+0000 mon.a (mon.0) 306 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-07-15T06:41:26.809 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:41:26 smithi134 bash[24068]: audit 2024-07-15T06:41:25.852788+0000 mon.c (mon.2) 5 : audit [INF] from='osd.1 [v2:172.21.15.89:6810/1536075263,v1:172.21.15.89:6811/1536075263]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-07-15T06:41:26.809 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:41:26 smithi134 bash[24068]: audit 2024-07-15T06:41:25.853133+0000 mon.a (mon.0) 306 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-07-15T06:41:27.767 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:41:27 smithi089 bash[31372]: audit 2024-07-15T06:41:26.501911+0000 mon.a (mon.0) 307 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-07-15T06:41:27.767 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:41:27 smithi089 bash[31372]: cluster 2024-07-15T06:41:26.502081+0000 mon.a (mon.0) 308 : cluster [DBG] osdmap e11: 2 total, 1 up, 2 in 2024-07-15T06:41:27.767 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:41:27 smithi089 bash[31372]: audit 2024-07-15T06:41:26.502624+0000 mon.a (mon.0) 309 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-07-15T06:41:27.767 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:41:27 smithi089 bash[31372]: audit 2024-07-15T06:41:26.503843+0000 mon.c (mon.2) 6 : audit [INF] from='osd.1 [v2:172.21.15.89:6810/1536075263,v1:172.21.15.89:6811/1536075263]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2024-07-15T06:41:27.767 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:41:27 smithi089 bash[31372]: audit 2024-07-15T06:41:26.504612+0000 mon.a (mon.0) 310 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2024-07-15T06:41:27.767 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:41:27 smithi089 bash[31372]: cluster 2024-07-15T06:41:26.892643+0000 mgr.y (mgr.14150) 137 : cluster [DBG] pgmap v101: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-15T06:41:27.768 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:41:27 smithi089 bash[21793]: audit 2024-07-15T06:41:26.501911+0000 mon.a (mon.0) 307 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-07-15T06:41:27.768 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:41:27 smithi089 bash[21793]: cluster 2024-07-15T06:41:26.502081+0000 mon.a (mon.0) 308 : cluster [DBG] osdmap e11: 2 total, 1 up, 2 in 2024-07-15T06:41:27.768 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:41:27 smithi089 bash[21793]: audit 2024-07-15T06:41:26.502624+0000 mon.a (mon.0) 309 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-07-15T06:41:27.768 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:41:27 smithi089 bash[21793]: audit 2024-07-15T06:41:26.503843+0000 mon.c (mon.2) 6 : audit [INF] from='osd.1 [v2:172.21.15.89:6810/1536075263,v1:172.21.15.89:6811/1536075263]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2024-07-15T06:41:27.768 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:41:27 smithi089 bash[21793]: audit 2024-07-15T06:41:26.504612+0000 mon.a (mon.0) 310 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2024-07-15T06:41:27.768 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:41:27 smithi089 bash[21793]: cluster 2024-07-15T06:41:26.892643+0000 mgr.y (mgr.14150) 137 : cluster [DBG] pgmap v101: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-15T06:41:27.809 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:41:27 smithi134 bash[24068]: audit 2024-07-15T06:41:26.501911+0000 mon.a (mon.0) 307 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-07-15T06:41:27.809 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:41:27 smithi134 bash[24068]: cluster 2024-07-15T06:41:26.502081+0000 mon.a (mon.0) 308 : cluster [DBG] osdmap e11: 2 total, 1 up, 2 in 2024-07-15T06:41:27.809 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:41:27 smithi134 bash[24068]: audit 2024-07-15T06:41:26.502624+0000 mon.a (mon.0) 309 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-07-15T06:41:27.809 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:41:27 smithi134 bash[24068]: audit 2024-07-15T06:41:26.503843+0000 mon.c (mon.2) 6 : audit [INF] from='osd.1 [v2:172.21.15.89:6810/1536075263,v1:172.21.15.89:6811/1536075263]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2024-07-15T06:41:27.809 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:41:27 smithi134 bash[24068]: audit 2024-07-15T06:41:26.504612+0000 mon.a (mon.0) 310 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2024-07-15T06:41:27.809 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:41:27 smithi134 bash[24068]: cluster 2024-07-15T06:41:26.892643+0000 mgr.y (mgr.14150) 137 : cluster [DBG] pgmap v101: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-15T06:41:28.189 INFO:teuthology.orchestra.run.smithi089.stdout:Created osd(s) 1 on host 'smithi089' 2024-07-15T06:41:28.783 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:41:28 smithi089 bash[31372]: audit 2024-07-15T06:41:27.500629+0000 mon.a (mon.0) 311 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]': finished 2024-07-15T06:41:28.783 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:41:28 smithi089 bash[31372]: cluster 2024-07-15T06:41:27.500687+0000 mon.a (mon.0) 312 : cluster [DBG] osdmap e12: 2 total, 1 up, 2 in 2024-07-15T06:41:28.783 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:41:28 smithi089 bash[31372]: audit 2024-07-15T06:41:27.501032+0000 mon.a (mon.0) 313 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-07-15T06:41:28.783 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:41:28 smithi089 bash[31372]: audit 2024-07-15T06:41:27.507824+0000 mon.a (mon.0) 314 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-07-15T06:41:28.783 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:41:28 smithi089 bash[31372]: audit 2024-07-15T06:41:28.171820+0000 mon.a (mon.0) 315 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:41:28.783 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:41:28 smithi089 bash[31372]: audit 2024-07-15T06:41:28.178785+0000 mon.a (mon.0) 316 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:41:28.783 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:41:28 smithi089 bash[31372]: audit 2024-07-15T06:41:28.186229+0000 mon.a (mon.0) 317 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:41:28.784 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:41:28 smithi089 bash[21793]: audit 2024-07-15T06:41:27.500629+0000 mon.a (mon.0) 311 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]': finished 2024-07-15T06:41:28.784 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:41:28 smithi089 bash[21793]: cluster 2024-07-15T06:41:27.500687+0000 mon.a (mon.0) 312 : cluster [DBG] osdmap e12: 2 total, 1 up, 2 in 2024-07-15T06:41:28.784 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:41:28 smithi089 bash[21793]: audit 2024-07-15T06:41:27.501032+0000 mon.a (mon.0) 313 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-07-15T06:41:28.784 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:41:28 smithi089 bash[21793]: audit 2024-07-15T06:41:27.507824+0000 mon.a (mon.0) 314 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-07-15T06:41:28.784 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:41:28 smithi089 bash[21793]: audit 2024-07-15T06:41:28.171820+0000 mon.a (mon.0) 315 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:41:28.784 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:41:28 smithi089 bash[21793]: audit 2024-07-15T06:41:28.178785+0000 mon.a (mon.0) 316 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:41:28.784 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:41:28 smithi089 bash[21793]: audit 2024-07-15T06:41:28.186229+0000 mon.a (mon.0) 317 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:41:28.785 DEBUG:teuthology.orchestra.run.smithi089:osd.1> sudo journalctl -f -n 0 -u ceph-75ba02bc-4274-11ef-bcad-c7b262605968@osd.1.service 2024-07-15T06:41:28.791 INFO:tasks.cephadm:Deploying osd.2 on smithi089 with /dev/vg_nvme/lv_2... 2024-07-15T06:41:28.791 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-07-15T06:41:28.809 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:41:28 smithi134 bash[24068]: audit 2024-07-15T06:41:27.500629+0000 mon.a (mon.0) 311 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]': finished 2024-07-15T06:41:28.809 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:41:28 smithi134 bash[24068]: cluster 2024-07-15T06:41:27.500687+0000 mon.a (mon.0) 312 : cluster [DBG] osdmap e12: 2 total, 1 up, 2 in 2024-07-15T06:41:28.809 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:41:28 smithi134 bash[24068]: audit 2024-07-15T06:41:27.501032+0000 mon.a (mon.0) 313 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-07-15T06:41:28.809 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:41:28 smithi134 bash[24068]: audit 2024-07-15T06:41:27.507824+0000 mon.a (mon.0) 314 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-07-15T06:41:28.810 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:41:28 smithi134 bash[24068]: audit 2024-07-15T06:41:28.171820+0000 mon.a (mon.0) 315 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:41:28.810 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:41:28 smithi134 bash[24068]: audit 2024-07-15T06:41:28.178785+0000 mon.a (mon.0) 316 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:41:28.810 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:41:28 smithi134 bash[24068]: audit 2024-07-15T06:41:28.186229+0000 mon.a (mon.0) 317 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:41:29.809 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:41:29 smithi134 bash[24068]: cluster 2024-07-15T06:41:26.849479+0000 osd.1 (osd.1) 1 : cluster [DBG] purged_snaps scrub starts 2024-07-15T06:41:29.809 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:41:29 smithi134 bash[24068]: cluster 2024-07-15T06:41:26.849570+0000 osd.1 (osd.1) 2 : cluster [DBG] purged_snaps scrub ok 2024-07-15T06:41:29.809 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:41:29 smithi134 bash[24068]: audit 2024-07-15T06:41:28.512845+0000 mon.a (mon.0) 318 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-07-15T06:41:29.809 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:41:29 smithi134 bash[24068]: cluster 2024-07-15T06:41:28.517993+0000 mon.a (mon.0) 319 : cluster [INF] osd.1 [v2:172.21.15.89:6810/1536075263,v1:172.21.15.89:6811/1536075263] boot 2024-07-15T06:41:29.809 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:41:29 smithi134 bash[24068]: cluster 2024-07-15T06:41:28.518060+0000 mon.a (mon.0) 320 : cluster [DBG] osdmap e13: 2 total, 2 up, 2 in 2024-07-15T06:41:29.809 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:41:29 smithi134 bash[24068]: audit 2024-07-15T06:41:28.518232+0000 mon.a (mon.0) 321 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-07-15T06:41:29.809 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:41:29 smithi134 bash[24068]: cluster 2024-07-15T06:41:28.892967+0000 mgr.y (mgr.14150) 138 : cluster [DBG] pgmap v104: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-15T06:41:29.874 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:41:29 smithi089 bash[21793]: cluster 2024-07-15T06:41:26.849479+0000 osd.1 (osd.1) 1 : cluster [DBG] purged_snaps scrub starts 2024-07-15T06:41:29.874 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:41:29 smithi089 bash[21793]: cluster 2024-07-15T06:41:26.849570+0000 osd.1 (osd.1) 2 : cluster [DBG] purged_snaps scrub ok 2024-07-15T06:41:29.874 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:41:29 smithi089 bash[21793]: audit 2024-07-15T06:41:28.512845+0000 mon.a (mon.0) 318 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-07-15T06:41:29.874 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:41:29 smithi089 bash[21793]: cluster 2024-07-15T06:41:28.517993+0000 mon.a (mon.0) 319 : cluster [INF] osd.1 [v2:172.21.15.89:6810/1536075263,v1:172.21.15.89:6811/1536075263] boot 2024-07-15T06:41:29.874 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:41:29 smithi089 bash[21793]: cluster 2024-07-15T06:41:28.518060+0000 mon.a (mon.0) 320 : cluster [DBG] osdmap e13: 2 total, 2 up, 2 in 2024-07-15T06:41:29.875 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:41:29 smithi089 bash[21793]: audit 2024-07-15T06:41:28.518232+0000 mon.a (mon.0) 321 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-07-15T06:41:29.875 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:41:29 smithi089 bash[21793]: cluster 2024-07-15T06:41:28.892967+0000 mgr.y (mgr.14150) 138 : cluster [DBG] pgmap v104: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-15T06:41:29.875 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:41:29 smithi089 bash[31372]: cluster 2024-07-15T06:41:26.849479+0000 osd.1 (osd.1) 1 : cluster [DBG] purged_snaps scrub starts 2024-07-15T06:41:29.875 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:41:29 smithi089 bash[31372]: cluster 2024-07-15T06:41:26.849570+0000 osd.1 (osd.1) 2 : cluster [DBG] purged_snaps scrub ok 2024-07-15T06:41:29.875 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:41:29 smithi089 bash[31372]: audit 2024-07-15T06:41:28.512845+0000 mon.a (mon.0) 318 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-07-15T06:41:29.875 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:41:29 smithi089 bash[31372]: cluster 2024-07-15T06:41:28.517993+0000 mon.a (mon.0) 319 : cluster [INF] osd.1 [v2:172.21.15.89:6810/1536075263,v1:172.21.15.89:6811/1536075263] boot 2024-07-15T06:41:29.875 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:41:29 smithi089 bash[31372]: cluster 2024-07-15T06:41:28.518060+0000 mon.a (mon.0) 320 : cluster [DBG] osdmap e13: 2 total, 2 up, 2 in 2024-07-15T06:41:29.875 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:41:29 smithi089 bash[31372]: audit 2024-07-15T06:41:28.518232+0000 mon.a (mon.0) 321 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-07-15T06:41:29.875 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:41:29 smithi089 bash[31372]: cluster 2024-07-15T06:41:28.892967+0000 mgr.y (mgr.14150) 138 : cluster [DBG] pgmap v104: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-15T06:41:30.809 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:41:30 smithi134 bash[24068]: cluster 2024-07-15T06:41:29.532907+0000 mon.a (mon.0) 322 : cluster [DBG] osdmap e14: 2 total, 2 up, 2 in 2024-07-15T06:41:30.874 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:41:30 smithi089 bash[21793]: cluster 2024-07-15T06:41:29.532907+0000 mon.a (mon.0) 322 : cluster [DBG] osdmap e14: 2 total, 2 up, 2 in 2024-07-15T06:41:30.874 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:41:30 smithi089 bash[31372]: cluster 2024-07-15T06:41:29.532907+0000 mon.a (mon.0) 322 : cluster [DBG] osdmap e14: 2 total, 2 up, 2 in 2024-07-15T06:41:31.809 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:41:31 smithi134 bash[24068]: cluster 2024-07-15T06:41:30.893406+0000 mgr.y (mgr.14150) 139 : cluster [DBG] pgmap v106: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-15T06:41:31.873 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:41:31 smithi089 bash[21793]: cluster 2024-07-15T06:41:30.893406+0000 mgr.y (mgr.14150) 139 : cluster [DBG] pgmap v106: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-15T06:41:31.874 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:41:31 smithi089 bash[31372]: cluster 2024-07-15T06:41:30.893406+0000 mgr.y (mgr.14150) 139 : cluster [DBG] pgmap v106: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-15T06:41:32.952 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T06:41:34.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:41:33 smithi134 bash[24068]: cluster 2024-07-15T06:41:32.893874+0000 mgr.y (mgr.14150) 140 : cluster [DBG] pgmap v107: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-15T06:41:34.373 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:41:33 smithi089 bash[21793]: cluster 2024-07-15T06:41:32.893874+0000 mgr.y (mgr.14150) 140 : cluster [DBG] pgmap v107: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-15T06:41:34.374 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:41:33 smithi089 bash[31372]: cluster 2024-07-15T06:41:32.893874+0000 mgr.y (mgr.14150) 140 : cluster [DBG] pgmap v107: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-15T06:41:36.124 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:41:35 smithi089 bash[31372]: cluster 2024-07-15T06:41:34.894366+0000 mgr.y (mgr.14150) 141 : cluster [DBG] pgmap v108: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-15T06:41:36.124 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:41:35 smithi089 bash[21793]: cluster 2024-07-15T06:41:34.894366+0000 mgr.y (mgr.14150) 141 : cluster [DBG] pgmap v108: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-15T06:41:36.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:41:35 smithi134 bash[24068]: cluster 2024-07-15T06:41:34.894366+0000 mgr.y (mgr.14150) 141 : cluster [DBG] pgmap v108: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-15T06:41:37.257 INFO:teuthology.orchestra.run.smithi089.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-07-15T06:41:37.409 INFO:teuthology.orchestra.run.smithi089.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-07-15T06:41:37.424 INFO:teuthology.orchestra.run.smithi089.stderr: stderr: 10+0 records in 2024-07-15T06:41:37.424 INFO:teuthology.orchestra.run.smithi089.stderr:10+0 records out 2024-07-15T06:41:37.425 INFO:teuthology.orchestra.run.smithi089.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0135769 s, 772 MB/s 2024-07-15T06:41:37.425 INFO:teuthology.orchestra.run.smithi089.stderr:--> Zapping successful for: 2024-07-15T06:41:38.205 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:41:37 smithi089 bash[21793]: cluster 2024-07-15T06:41:36.894679+0000 mgr.y (mgr.14150) 142 : cluster [DBG] pgmap v109: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-15T06:41:38.205 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:41:37 smithi089 bash[31372]: cluster 2024-07-15T06:41:36.894679+0000 mgr.y (mgr.14150) 142 : cluster [DBG] pgmap v109: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-15T06:41:38.207 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph orch daemon add osd smithi089:vg_nvme/lv_2 2024-07-15T06:41:38.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:41:37 smithi134 bash[24068]: cluster 2024-07-15T06:41:36.894679+0000 mgr.y (mgr.14150) 142 : cluster [DBG] pgmap v109: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-15T06:41:39.809 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:41:39 smithi134 bash[24068]: cephadm 2024-07-15T06:41:38.530795+0000 mgr.y (mgr.14150) 143 : cephadm [INF] Detected new or changed devices on smithi089 2024-07-15T06:41:39.809 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:41:39 smithi134 bash[24068]: audit 2024-07-15T06:41:38.538908+0000 mon.a (mon.0) 323 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:41:39.809 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:41:39 smithi134 bash[24068]: audit 2024-07-15T06:41:38.547265+0000 mon.a (mon.0) 324 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:41:39.809 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:41:39 smithi134 bash[24068]: audit 2024-07-15T06:41:38.549152+0000 mon.a (mon.0) 325 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:41:39.810 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:41:39 smithi134 bash[24068]: audit 2024-07-15T06:41:38.550788+0000 mon.a (mon.0) 326 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:41:39.810 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:41:39 smithi134 bash[24068]: cephadm 2024-07-15T06:41:38.551966+0000 mgr.y (mgr.14150) 144 : cephadm [INF] Adjusting osd_memory_target on smithi089 to 8120M 2024-07-15T06:41:39.810 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:41:39 smithi134 bash[24068]: audit 2024-07-15T06:41:38.560026+0000 mon.a (mon.0) 327 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:41:39.810 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:41:39 smithi134 bash[24068]: audit 2024-07-15T06:41:38.562518+0000 mon.a (mon.0) 328 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:41:39.810 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:41:39 smithi134 bash[24068]: audit 2024-07-15T06:41:38.564283+0000 mon.a (mon.0) 329 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:41:39.810 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:41:39 smithi134 bash[24068]: audit 2024-07-15T06:41:38.574774+0000 mon.a (mon.0) 330 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:41:39.810 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:41:39 smithi134 bash[24068]: cluster 2024-07-15T06:41:38.895107+0000 mgr.y (mgr.14150) 145 : cluster [DBG] pgmap v110: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-15T06:41:39.874 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:41:39 smithi089 bash[21793]: cephadm 2024-07-15T06:41:38.530795+0000 mgr.y (mgr.14150) 143 : cephadm [INF] Detected new or changed devices on smithi089 2024-07-15T06:41:39.874 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:41:39 smithi089 bash[21793]: audit 2024-07-15T06:41:38.538908+0000 mon.a (mon.0) 323 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:41:39.874 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:41:39 smithi089 bash[21793]: audit 2024-07-15T06:41:38.547265+0000 mon.a (mon.0) 324 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:41:39.874 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:41:39 smithi089 bash[21793]: audit 2024-07-15T06:41:38.549152+0000 mon.a (mon.0) 325 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:41:39.874 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:41:39 smithi089 bash[21793]: audit 2024-07-15T06:41:38.550788+0000 mon.a (mon.0) 326 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:41:39.874 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:41:39 smithi089 bash[21793]: cephadm 2024-07-15T06:41:38.551966+0000 mgr.y (mgr.14150) 144 : cephadm [INF] Adjusting osd_memory_target on smithi089 to 8120M 2024-07-15T06:41:39.874 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:41:39 smithi089 bash[21793]: audit 2024-07-15T06:41:38.560026+0000 mon.a (mon.0) 327 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:41:39.875 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:41:39 smithi089 bash[21793]: audit 2024-07-15T06:41:38.562518+0000 mon.a (mon.0) 328 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:41:39.875 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:41:39 smithi089 bash[21793]: audit 2024-07-15T06:41:38.564283+0000 mon.a (mon.0) 329 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:41:39.875 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:41:39 smithi089 bash[21793]: audit 2024-07-15T06:41:38.574774+0000 mon.a (mon.0) 330 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:41:39.875 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:41:39 smithi089 bash[21793]: cluster 2024-07-15T06:41:38.895107+0000 mgr.y (mgr.14150) 145 : cluster [DBG] pgmap v110: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-15T06:41:39.875 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:41:39 smithi089 bash[31372]: cephadm 2024-07-15T06:41:38.530795+0000 mgr.y (mgr.14150) 143 : cephadm [INF] Detected new or changed devices on smithi089 2024-07-15T06:41:39.875 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:41:39 smithi089 bash[31372]: audit 2024-07-15T06:41:38.538908+0000 mon.a (mon.0) 323 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:41:39.875 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:41:39 smithi089 bash[31372]: audit 2024-07-15T06:41:38.547265+0000 mon.a (mon.0) 324 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:41:39.875 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:41:39 smithi089 bash[31372]: audit 2024-07-15T06:41:38.549152+0000 mon.a (mon.0) 325 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:41:39.875 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:41:39 smithi089 bash[31372]: audit 2024-07-15T06:41:38.550788+0000 mon.a (mon.0) 326 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:41:39.875 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:41:39 smithi089 bash[31372]: cephadm 2024-07-15T06:41:38.551966+0000 mgr.y (mgr.14150) 144 : cephadm [INF] Adjusting osd_memory_target on smithi089 to 8120M 2024-07-15T06:41:39.875 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:41:39 smithi089 bash[31372]: audit 2024-07-15T06:41:38.560026+0000 mon.a (mon.0) 327 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:41:39.875 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:41:39 smithi089 bash[31372]: audit 2024-07-15T06:41:38.562518+0000 mon.a (mon.0) 328 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:41:39.876 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:41:39 smithi089 bash[31372]: audit 2024-07-15T06:41:38.564283+0000 mon.a (mon.0) 329 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:41:39.876 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:41:39 smithi089 bash[31372]: audit 2024-07-15T06:41:38.574774+0000 mon.a (mon.0) 330 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:41:39.876 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:41:39 smithi089 bash[31372]: cluster 2024-07-15T06:41:38.895107+0000 mgr.y (mgr.14150) 145 : cluster [DBG] pgmap v110: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-15T06:41:42.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:41:41 smithi134 bash[24068]: cluster 2024-07-15T06:41:40.895650+0000 mgr.y (mgr.14150) 146 : cluster [DBG] pgmap v111: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-15T06:41:42.374 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:41:41 smithi089 bash[21793]: cluster 2024-07-15T06:41:40.895650+0000 mgr.y (mgr.14150) 146 : cluster [DBG] pgmap v111: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-15T06:41:42.374 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:41:41 smithi089 bash[31372]: cluster 2024-07-15T06:41:40.895650+0000 mgr.y (mgr.14150) 146 : cluster [DBG] pgmap v111: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-15T06:41:42.951 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T06:41:44.288 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:41:43 smithi089 bash[21793]: cluster 2024-07-15T06:41:42.896132+0000 mgr.y (mgr.14150) 147 : cluster [DBG] pgmap v112: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-15T06:41:44.288 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:41:43 smithi089 bash[31372]: cluster 2024-07-15T06:41:42.896132+0000 mgr.y (mgr.14150) 147 : cluster [DBG] pgmap v112: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-15T06:41:44.308 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:41:43 smithi134 bash[24068]: cluster 2024-07-15T06:41:42.896132+0000 mgr.y (mgr.14150) 147 : cluster [DBG] pgmap v112: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-15T06:41:46.123 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:41:45 smithi089 bash[21793]: cluster 2024-07-15T06:41:44.896665+0000 mgr.y (mgr.14150) 148 : cluster [DBG] pgmap v113: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-15T06:41:46.124 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:41:45 smithi089 bash[31372]: cluster 2024-07-15T06:41:44.896665+0000 mgr.y (mgr.14150) 148 : cluster [DBG] pgmap v113: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-15T06:41:46.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:41:45 smithi134 bash[24068]: cluster 2024-07-15T06:41:44.896665+0000 mgr.y (mgr.14150) 148 : cluster [DBG] pgmap v113: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-15T06:41:47.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:41:46 smithi134 bash[24068]: audit 2024-07-15T06:41:46.092107+0000 mgr.y (mgr.14150) 149 : audit [DBG] from='client.24167 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi089:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T06:41:47.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:41:46 smithi134 bash[24068]: audit 2024-07-15T06:41:46.094638+0000 mon.a (mon.0) 331 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-07-15T06:41:47.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:41:46 smithi134 bash[24068]: audit 2024-07-15T06:41:46.097997+0000 mon.a (mon.0) 332 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-07-15T06:41:47.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:41:46 smithi134 bash[24068]: audit 2024-07-15T06:41:46.099101+0000 mon.a (mon.0) 333 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:41:47.373 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:41:46 smithi089 bash[21793]: audit 2024-07-15T06:41:46.092107+0000 mgr.y (mgr.14150) 149 : audit [DBG] from='client.24167 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi089:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T06:41:47.374 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:41:46 smithi089 bash[21793]: audit 2024-07-15T06:41:46.094638+0000 mon.a (mon.0) 331 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-07-15T06:41:47.374 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:41:46 smithi089 bash[21793]: audit 2024-07-15T06:41:46.097997+0000 mon.a (mon.0) 332 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-07-15T06:41:47.374 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:41:46 smithi089 bash[21793]: audit 2024-07-15T06:41:46.099101+0000 mon.a (mon.0) 333 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:41:47.374 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:41:46 smithi089 bash[31372]: audit 2024-07-15T06:41:46.092107+0000 mgr.y (mgr.14150) 149 : audit [DBG] from='client.24167 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi089:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T06:41:47.374 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:41:46 smithi089 bash[31372]: audit 2024-07-15T06:41:46.094638+0000 mon.a (mon.0) 331 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-07-15T06:41:47.374 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:41:46 smithi089 bash[31372]: audit 2024-07-15T06:41:46.097997+0000 mon.a (mon.0) 332 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-07-15T06:41:47.374 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:41:46 smithi089 bash[31372]: audit 2024-07-15T06:41:46.099101+0000 mon.a (mon.0) 333 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:41:48.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:41:47 smithi134 bash[24068]: cluster 2024-07-15T06:41:46.897147+0000 mgr.y (mgr.14150) 150 : cluster [DBG] pgmap v114: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-15T06:41:48.373 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:41:47 smithi089 bash[21793]: cluster 2024-07-15T06:41:46.897147+0000 mgr.y (mgr.14150) 150 : cluster [DBG] pgmap v114: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-15T06:41:48.374 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:41:47 smithi089 bash[31372]: cluster 2024-07-15T06:41:46.897147+0000 mgr.y (mgr.14150) 150 : cluster [DBG] pgmap v114: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-15T06:41:50.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:41:49 smithi134 bash[24068]: cluster 2024-07-15T06:41:48.897595+0000 mgr.y (mgr.14150) 151 : cluster [DBG] pgmap v115: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-15T06:41:50.373 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:41:49 smithi089 bash[21793]: cluster 2024-07-15T06:41:48.897595+0000 mgr.y (mgr.14150) 151 : cluster [DBG] pgmap v115: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-15T06:41:50.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:41:49 smithi089 bash[31372]: cluster 2024-07-15T06:41:48.897595+0000 mgr.y (mgr.14150) 151 : cluster [DBG] pgmap v115: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-15T06:41:52.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:41:51 smithi134 bash[24068]: cluster 2024-07-15T06:41:50.898033+0000 mgr.y (mgr.14150) 152 : cluster [DBG] pgmap v116: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-15T06:41:52.328 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:41:51 smithi089 bash[21793]: cluster 2024-07-15T06:41:50.898033+0000 mgr.y (mgr.14150) 152 : cluster [DBG] pgmap v116: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-15T06:41:52.329 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:41:51 smithi089 bash[31372]: cluster 2024-07-15T06:41:50.898033+0000 mgr.y (mgr.14150) 152 : cluster [DBG] pgmap v116: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-15T06:41:54.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:41:53 smithi134 bash[24068]: cluster 2024-07-15T06:41:52.898488+0000 mgr.y (mgr.14150) 153 : cluster [DBG] pgmap v117: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-15T06:41:54.373 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:41:53 smithi089 bash[21793]: cluster 2024-07-15T06:41:52.898488+0000 mgr.y (mgr.14150) 153 : cluster [DBG] pgmap v117: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-15T06:41:54.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:41:53 smithi089 bash[31372]: cluster 2024-07-15T06:41:52.898488+0000 mgr.y (mgr.14150) 153 : cluster [DBG] pgmap v117: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-15T06:41:55.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:41:54 smithi134 bash[24068]: audit 2024-07-15T06:41:54.361232+0000 mon.c (mon.2) 7 : audit [INF] from='client.? 172.21.15.89:0/222271110' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "60cde686-83bb-4ca9-a0f1-6668710e16f1"}]: dispatch 2024-07-15T06:41:55.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:41:54 smithi134 bash[24068]: audit 2024-07-15T06:41:54.361880+0000 mon.a (mon.0) 334 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "60cde686-83bb-4ca9-a0f1-6668710e16f1"}]: dispatch 2024-07-15T06:41:55.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:41:54 smithi134 bash[24068]: audit 2024-07-15T06:41:54.369109+0000 mon.a (mon.0) 335 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "60cde686-83bb-4ca9-a0f1-6668710e16f1"}]': finished 2024-07-15T06:41:55.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:41:54 smithi134 bash[24068]: cluster 2024-07-15T06:41:54.369284+0000 mon.a (mon.0) 336 : cluster [DBG] osdmap e15: 3 total, 2 up, 3 in 2024-07-15T06:41:55.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:41:54 smithi134 bash[24068]: audit 2024-07-15T06:41:54.369520+0000 mon.a (mon.0) 337 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-07-15T06:41:55.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:41:54 smithi089 bash[31372]: audit 2024-07-15T06:41:54.361232+0000 mon.c (mon.2) 7 : audit [INF] from='client.? 172.21.15.89:0/222271110' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "60cde686-83bb-4ca9-a0f1-6668710e16f1"}]: dispatch 2024-07-15T06:41:55.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:41:54 smithi089 bash[31372]: audit 2024-07-15T06:41:54.361880+0000 mon.a (mon.0) 334 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "60cde686-83bb-4ca9-a0f1-6668710e16f1"}]: dispatch 2024-07-15T06:41:55.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:41:54 smithi089 bash[31372]: audit 2024-07-15T06:41:54.369109+0000 mon.a (mon.0) 335 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "60cde686-83bb-4ca9-a0f1-6668710e16f1"}]': finished 2024-07-15T06:41:55.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:41:54 smithi089 bash[31372]: cluster 2024-07-15T06:41:54.369284+0000 mon.a (mon.0) 336 : cluster [DBG] osdmap e15: 3 total, 2 up, 3 in 2024-07-15T06:41:55.374 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:41:54 smithi089 bash[31372]: audit 2024-07-15T06:41:54.369520+0000 mon.a (mon.0) 337 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-07-15T06:41:55.374 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:41:54 smithi089 bash[21793]: audit 2024-07-15T06:41:54.361232+0000 mon.c (mon.2) 7 : audit [INF] from='client.? 172.21.15.89:0/222271110' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "60cde686-83bb-4ca9-a0f1-6668710e16f1"}]: dispatch 2024-07-15T06:41:55.374 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:41:54 smithi089 bash[21793]: audit 2024-07-15T06:41:54.361880+0000 mon.a (mon.0) 334 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "60cde686-83bb-4ca9-a0f1-6668710e16f1"}]: dispatch 2024-07-15T06:41:55.374 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:41:54 smithi089 bash[21793]: audit 2024-07-15T06:41:54.369109+0000 mon.a (mon.0) 335 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "60cde686-83bb-4ca9-a0f1-6668710e16f1"}]': finished 2024-07-15T06:41:55.374 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:41:54 smithi089 bash[21793]: cluster 2024-07-15T06:41:54.369284+0000 mon.a (mon.0) 336 : cluster [DBG] osdmap e15: 3 total, 2 up, 3 in 2024-07-15T06:41:55.374 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:41:54 smithi089 bash[21793]: audit 2024-07-15T06:41:54.369520+0000 mon.a (mon.0) 337 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-07-15T06:41:56.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:41:55 smithi134 bash[24068]: cluster 2024-07-15T06:41:54.898917+0000 mgr.y (mgr.14150) 154 : cluster [DBG] pgmap v119: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-15T06:41:56.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:41:55 smithi134 bash[24068]: audit 2024-07-15T06:41:55.011808+0000 mon.a (mon.0) 338 : audit [DBG] from='client.? 172.21.15.89:0/3957790403' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-07-15T06:41:56.373 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:41:55 smithi089 bash[21793]: cluster 2024-07-15T06:41:54.898917+0000 mgr.y (mgr.14150) 154 : cluster [DBG] pgmap v119: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-15T06:41:56.373 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:41:55 smithi089 bash[21793]: audit 2024-07-15T06:41:55.011808+0000 mon.a (mon.0) 338 : audit [DBG] from='client.? 172.21.15.89:0/3957790403' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-07-15T06:41:56.374 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:41:55 smithi089 bash[31372]: cluster 2024-07-15T06:41:54.898917+0000 mgr.y (mgr.14150) 154 : cluster [DBG] pgmap v119: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-15T06:41:56.374 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:41:55 smithi089 bash[31372]: audit 2024-07-15T06:41:55.011808+0000 mon.a (mon.0) 338 : audit [DBG] from='client.? 172.21.15.89:0/3957790403' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-07-15T06:41:58.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:41:57 smithi134 bash[24068]: cluster 2024-07-15T06:41:56.899400+0000 mgr.y (mgr.14150) 155 : cluster [DBG] pgmap v120: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-15T06:41:58.373 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:41:57 smithi089 bash[21793]: cluster 2024-07-15T06:41:56.899400+0000 mgr.y (mgr.14150) 155 : cluster [DBG] pgmap v120: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-15T06:41:58.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:41:57 smithi089 bash[31372]: cluster 2024-07-15T06:41:56.899400+0000 mgr.y (mgr.14150) 155 : cluster [DBG] pgmap v120: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-15T06:42:00.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:41:59 smithi134 bash[24068]: cluster 2024-07-15T06:41:58.899797+0000 mgr.y (mgr.14150) 156 : cluster [DBG] pgmap v121: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-15T06:42:00.373 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:41:59 smithi089 bash[21793]: cluster 2024-07-15T06:41:58.899797+0000 mgr.y (mgr.14150) 156 : cluster [DBG] pgmap v121: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-15T06:42:00.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:41:59 smithi089 bash[31372]: cluster 2024-07-15T06:41:58.899797+0000 mgr.y (mgr.14150) 156 : cluster [DBG] pgmap v121: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-15T06:42:02.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:01 smithi134 bash[24068]: cluster 2024-07-15T06:42:00.900156+0000 mgr.y (mgr.14150) 157 : cluster [DBG] pgmap v122: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-15T06:42:02.373 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:01 smithi089 bash[21793]: cluster 2024-07-15T06:42:00.900156+0000 mgr.y (mgr.14150) 157 : cluster [DBG] pgmap v122: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-15T06:42:02.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:01 smithi089 bash[31372]: cluster 2024-07-15T06:42:00.900156+0000 mgr.y (mgr.14150) 157 : cluster [DBG] pgmap v122: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-15T06:42:04.285 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:03 smithi089 bash[21793]: cluster 2024-07-15T06:42:02.900567+0000 mgr.y (mgr.14150) 158 : cluster [DBG] pgmap v123: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-15T06:42:04.285 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:03 smithi089 bash[31372]: cluster 2024-07-15T06:42:02.900567+0000 mgr.y (mgr.14150) 158 : cluster [DBG] pgmap v123: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-15T06:42:04.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:03 smithi134 bash[24068]: cluster 2024-07-15T06:42:02.900567+0000 mgr.y (mgr.14150) 158 : cluster [DBG] pgmap v123: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-15T06:42:06.268 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:06 smithi089 bash[21793]: cluster 2024-07-15T06:42:04.901012+0000 mgr.y (mgr.14150) 159 : cluster [DBG] pgmap v124: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-15T06:42:06.268 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:06 smithi089 bash[31372]: cluster 2024-07-15T06:42:04.901012+0000 mgr.y (mgr.14150) 159 : cluster [DBG] pgmap v124: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-15T06:42:06.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:05 smithi134 bash[24068]: cluster 2024-07-15T06:42:04.901012+0000 mgr.y (mgr.14150) 159 : cluster [DBG] pgmap v124: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-15T06:42:07.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:07 smithi134 bash[24068]: audit 2024-07-15T06:42:06.690469+0000 mon.a (mon.0) 339 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-07-15T06:42:07.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:07 smithi134 bash[24068]: audit 2024-07-15T06:42:06.692011+0000 mon.a (mon.0) 340 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:42:07.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:07 smithi089 bash[31372]: audit 2024-07-15T06:42:06.690469+0000 mon.a (mon.0) 339 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-07-15T06:42:07.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:07 smithi089 bash[31372]: audit 2024-07-15T06:42:06.692011+0000 mon.a (mon.0) 340 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:42:07.373 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:07 smithi089 bash[21793]: audit 2024-07-15T06:42:06.690469+0000 mon.a (mon.0) 339 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-07-15T06:42:07.373 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:07 smithi089 bash[21793]: audit 2024-07-15T06:42:06.692011+0000 mon.a (mon.0) 340 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:42:08.267 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:08 smithi089 bash[21793]: cephadm 2024-07-15T06:42:06.693250+0000 mgr.y (mgr.14150) 160 : cephadm [INF] Deploying daemon osd.2 on smithi089 2024-07-15T06:42:08.267 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:08 smithi089 bash[21793]: cluster 2024-07-15T06:42:06.901321+0000 mgr.y (mgr.14150) 161 : cluster [DBG] pgmap v125: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-15T06:42:08.268 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:08 smithi089 bash[31372]: cephadm 2024-07-15T06:42:06.693250+0000 mgr.y (mgr.14150) 160 : cephadm [INF] Deploying daemon osd.2 on smithi089 2024-07-15T06:42:08.268 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:08 smithi089 bash[31372]: cluster 2024-07-15T06:42:06.901321+0000 mgr.y (mgr.14150) 161 : cluster [DBG] pgmap v125: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-15T06:42:08.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:08 smithi134 bash[24068]: cephadm 2024-07-15T06:42:06.693250+0000 mgr.y (mgr.14150) 160 : cephadm [INF] Deploying daemon osd.2 on smithi089 2024-07-15T06:42:08.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:08 smithi134 bash[24068]: cluster 2024-07-15T06:42:06.901321+0000 mgr.y (mgr.14150) 161 : cluster [DBG] pgmap v125: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-15T06:42:10.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:10 smithi134 bash[24068]: cluster 2024-07-15T06:42:08.901782+0000 mgr.y (mgr.14150) 162 : cluster [DBG] pgmap v126: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-15T06:42:10.373 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:10 smithi089 bash[21793]: cluster 2024-07-15T06:42:08.901782+0000 mgr.y (mgr.14150) 162 : cluster [DBG] pgmap v126: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-15T06:42:10.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:10 smithi089 bash[31372]: cluster 2024-07-15T06:42:08.901782+0000 mgr.y (mgr.14150) 162 : cluster [DBG] pgmap v126: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-15T06:42:10.873 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:10 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:42:10.873 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:42:10 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:42:10.873 INFO:journalctl@ceph.osd.0.smithi089.stdout:Jul 15 06:42:10 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:42:10.874 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:10 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:42:10.874 INFO:journalctl@ceph.osd.1.smithi089.stdout:Jul 15 06:42:10 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:42:11.373 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:42:10 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:42:11.374 INFO:journalctl@ceph.osd.0.smithi089.stdout:Jul 15 06:42:10 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:42:11.374 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:10 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:42:11.374 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:10 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:42:11.375 INFO:journalctl@ceph.osd.1.smithi089.stdout:Jul 15 06:42:10 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:42:12.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:12 smithi134 bash[24068]: cluster 2024-07-15T06:42:10.902149+0000 mgr.y (mgr.14150) 163 : cluster [DBG] pgmap v127: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-15T06:42:12.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:12 smithi134 bash[24068]: audit 2024-07-15T06:42:11.084292+0000 mon.a (mon.0) 341 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:42:12.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:12 smithi134 bash[24068]: audit 2024-07-15T06:42:11.089462+0000 mon.a (mon.0) 342 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:42:12.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:12 smithi134 bash[24068]: audit 2024-07-15T06:42:11.094690+0000 mon.a (mon.0) 343 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:42:12.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:12 smithi089 bash[31372]: cluster 2024-07-15T06:42:10.902149+0000 mgr.y (mgr.14150) 163 : cluster [DBG] pgmap v127: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-15T06:42:12.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:12 smithi089 bash[31372]: audit 2024-07-15T06:42:11.084292+0000 mon.a (mon.0) 341 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:42:12.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:12 smithi089 bash[31372]: audit 2024-07-15T06:42:11.089462+0000 mon.a (mon.0) 342 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:42:12.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:12 smithi089 bash[31372]: audit 2024-07-15T06:42:11.094690+0000 mon.a (mon.0) 343 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:42:12.373 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:12 smithi089 bash[21793]: cluster 2024-07-15T06:42:10.902149+0000 mgr.y (mgr.14150) 163 : cluster [DBG] pgmap v127: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-15T06:42:12.373 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:12 smithi089 bash[21793]: audit 2024-07-15T06:42:11.084292+0000 mon.a (mon.0) 341 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:42:12.374 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:12 smithi089 bash[21793]: audit 2024-07-15T06:42:11.089462+0000 mon.a (mon.0) 342 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:42:12.374 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:12 smithi089 bash[21793]: audit 2024-07-15T06:42:11.094690+0000 mon.a (mon.0) 343 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:42:14.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:14 smithi134 bash[24068]: cluster 2024-07-15T06:42:12.902511+0000 mgr.y (mgr.14150) 164 : cluster [DBG] pgmap v128: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-15T06:42:14.373 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:14 smithi089 bash[21793]: cluster 2024-07-15T06:42:12.902511+0000 mgr.y (mgr.14150) 164 : cluster [DBG] pgmap v128: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-15T06:42:14.374 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:14 smithi089 bash[31372]: cluster 2024-07-15T06:42:12.902511+0000 mgr.y (mgr.14150) 164 : cluster [DBG] pgmap v128: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-15T06:42:16.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:16 smithi134 bash[24068]: cluster 2024-07-15T06:42:14.902925+0000 mgr.y (mgr.14150) 165 : cluster [DBG] pgmap v129: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-15T06:42:16.373 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:16 smithi089 bash[21793]: cluster 2024-07-15T06:42:14.902925+0000 mgr.y (mgr.14150) 165 : cluster [DBG] pgmap v129: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-15T06:42:16.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:16 smithi089 bash[31372]: cluster 2024-07-15T06:42:14.902925+0000 mgr.y (mgr.14150) 165 : cluster [DBG] pgmap v129: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-15T06:42:17.308 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:17 smithi089 bash[21793]: audit 2024-07-15T06:42:16.162241+0000 mon.a (mon.0) 344 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:42:17.308 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:17 smithi089 bash[21793]: audit 2024-07-15T06:42:16.168149+0000 mon.a (mon.0) 345 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:42:17.308 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:17 smithi089 bash[21793]: audit 2024-07-15T06:42:16.549293+0000 mon.a (mon.0) 346 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:42:17.308 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:17 smithi089 bash[21793]: audit 2024-07-15T06:42:16.550174+0000 mon.a (mon.0) 347 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:42:17.308 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:17 smithi089 bash[21793]: audit 2024-07-15T06:42:16.557738+0000 mon.a (mon.0) 348 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:42:17.309 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:17 smithi089 bash[31372]: audit 2024-07-15T06:42:16.162241+0000 mon.a (mon.0) 344 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:42:17.309 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:17 smithi089 bash[31372]: audit 2024-07-15T06:42:16.168149+0000 mon.a (mon.0) 345 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:42:17.309 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:17 smithi089 bash[31372]: audit 2024-07-15T06:42:16.549293+0000 mon.a (mon.0) 346 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:42:17.309 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:17 smithi089 bash[31372]: audit 2024-07-15T06:42:16.550174+0000 mon.a (mon.0) 347 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:42:17.309 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:17 smithi089 bash[31372]: audit 2024-07-15T06:42:16.557738+0000 mon.a (mon.0) 348 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:42:17.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:17 smithi134 bash[24068]: audit 2024-07-15T06:42:16.162241+0000 mon.a (mon.0) 344 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:42:17.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:17 smithi134 bash[24068]: audit 2024-07-15T06:42:16.168149+0000 mon.a (mon.0) 345 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:42:17.560 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:17 smithi134 bash[24068]: audit 2024-07-15T06:42:16.549293+0000 mon.a (mon.0) 346 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:42:17.560 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:17 smithi134 bash[24068]: audit 2024-07-15T06:42:16.550174+0000 mon.a (mon.0) 347 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:42:17.560 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:17 smithi134 bash[24068]: audit 2024-07-15T06:42:16.557738+0000 mon.a (mon.0) 348 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:42:18.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:18 smithi134 bash[24068]: cluster 2024-07-15T06:42:16.903384+0000 mgr.y (mgr.14150) 166 : cluster [DBG] pgmap v130: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-15T06:42:18.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:18 smithi134 bash[24068]: audit 2024-07-15T06:42:17.718586+0000 mon.a (mon.0) 349 : audit [INF] from='osd.2 [v2:172.21.15.89:6818/3236432318,v1:172.21.15.89:6819/3236432318]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-07-15T06:42:18.623 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:18 smithi089 bash[21793]: cluster 2024-07-15T06:42:16.903384+0000 mgr.y (mgr.14150) 166 : cluster [DBG] pgmap v130: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-15T06:42:18.623 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:18 smithi089 bash[21793]: audit 2024-07-15T06:42:17.718586+0000 mon.a (mon.0) 349 : audit [INF] from='osd.2 [v2:172.21.15.89:6818/3236432318,v1:172.21.15.89:6819/3236432318]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-07-15T06:42:18.623 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:18 smithi089 bash[31372]: cluster 2024-07-15T06:42:16.903384+0000 mgr.y (mgr.14150) 166 : cluster [DBG] pgmap v130: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-15T06:42:18.623 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:18 smithi089 bash[31372]: audit 2024-07-15T06:42:17.718586+0000 mon.a (mon.0) 349 : audit [INF] from='osd.2 [v2:172.21.15.89:6818/3236432318,v1:172.21.15.89:6819/3236432318]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-07-15T06:42:19.715 INFO:teuthology.orchestra.run.smithi089.stdout:Created osd(s) 2 on host 'smithi089' 2024-07-15T06:42:19.873 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:19 smithi089 bash[31372]: audit 2024-07-15T06:42:18.570145+0000 mon.a (mon.0) 350 : audit [INF] from='osd.2 [v2:172.21.15.89:6818/3236432318,v1:172.21.15.89:6819/3236432318]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-07-15T06:42:19.873 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:19 smithi089 bash[31372]: cluster 2024-07-15T06:42:18.570299+0000 mon.a (mon.0) 351 : cluster [DBG] osdmap e16: 3 total, 2 up, 3 in 2024-07-15T06:42:19.873 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:19 smithi089 bash[31372]: audit 2024-07-15T06:42:18.570600+0000 mon.a (mon.0) 352 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-07-15T06:42:19.873 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:19 smithi089 bash[31372]: audit 2024-07-15T06:42:18.571687+0000 mon.a (mon.0) 353 : audit [INF] from='osd.2 [v2:172.21.15.89:6818/3236432318,v1:172.21.15.89:6819/3236432318]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2024-07-15T06:42:19.873 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:19 smithi089 bash[31372]: cluster 2024-07-15T06:42:18.903775+0000 mgr.y (mgr.14150) 167 : cluster [DBG] pgmap v132: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-15T06:42:19.873 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:19 smithi089 bash[21793]: audit 2024-07-15T06:42:18.570145+0000 mon.a (mon.0) 350 : audit [INF] from='osd.2 [v2:172.21.15.89:6818/3236432318,v1:172.21.15.89:6819/3236432318]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-07-15T06:42:19.874 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:19 smithi089 bash[21793]: cluster 2024-07-15T06:42:18.570299+0000 mon.a (mon.0) 351 : cluster [DBG] osdmap e16: 3 total, 2 up, 3 in 2024-07-15T06:42:19.874 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:19 smithi089 bash[21793]: audit 2024-07-15T06:42:18.570600+0000 mon.a (mon.0) 352 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-07-15T06:42:19.874 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:19 smithi089 bash[21793]: audit 2024-07-15T06:42:18.571687+0000 mon.a (mon.0) 353 : audit [INF] from='osd.2 [v2:172.21.15.89:6818/3236432318,v1:172.21.15.89:6819/3236432318]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2024-07-15T06:42:19.874 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:19 smithi089 bash[21793]: cluster 2024-07-15T06:42:18.903775+0000 mgr.y (mgr.14150) 167 : cluster [DBG] pgmap v132: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-15T06:42:20.059 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:19 smithi134 bash[24068]: audit 2024-07-15T06:42:18.570145+0000 mon.a (mon.0) 350 : audit [INF] from='osd.2 [v2:172.21.15.89:6818/3236432318,v1:172.21.15.89:6819/3236432318]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-07-15T06:42:20.059 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:19 smithi134 bash[24068]: cluster 2024-07-15T06:42:18.570299+0000 mon.a (mon.0) 351 : cluster [DBG] osdmap e16: 3 total, 2 up, 3 in 2024-07-15T06:42:20.059 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:19 smithi134 bash[24068]: audit 2024-07-15T06:42:18.570600+0000 mon.a (mon.0) 352 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-07-15T06:42:20.059 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:19 smithi134 bash[24068]: audit 2024-07-15T06:42:18.571687+0000 mon.a (mon.0) 353 : audit [INF] from='osd.2 [v2:172.21.15.89:6818/3236432318,v1:172.21.15.89:6819/3236432318]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2024-07-15T06:42:20.060 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:19 smithi134 bash[24068]: cluster 2024-07-15T06:42:18.903775+0000 mgr.y (mgr.14150) 167 : cluster [DBG] pgmap v132: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-15T06:42:20.382 DEBUG:teuthology.orchestra.run.smithi089:osd.2> sudo journalctl -f -n 0 -u ceph-75ba02bc-4274-11ef-bcad-c7b262605968@osd.2.service 2024-07-15T06:42:20.385 INFO:tasks.cephadm:Deploying osd.3 on smithi089 with /dev/vg_nvme/lv_1... 2024-07-15T06:42:20.385 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-07-15T06:42:20.873 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:20 smithi089 bash[21793]: audit 2024-07-15T06:42:19.582416+0000 mon.a (mon.0) 354 : audit [INF] from='osd.2 [v2:172.21.15.89:6818/3236432318,v1:172.21.15.89:6819/3236432318]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]': finished 2024-07-15T06:42:20.874 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:20 smithi089 bash[21793]: cluster 2024-07-15T06:42:19.582584+0000 mon.a (mon.0) 355 : cluster [DBG] osdmap e17: 3 total, 2 up, 3 in 2024-07-15T06:42:20.874 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:20 smithi089 bash[21793]: audit 2024-07-15T06:42:19.583022+0000 mon.a (mon.0) 356 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-07-15T06:42:20.874 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:20 smithi089 bash[21793]: audit 2024-07-15T06:42:19.589606+0000 mon.a (mon.0) 357 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-07-15T06:42:20.874 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:20 smithi089 bash[21793]: audit 2024-07-15T06:42:19.699585+0000 mon.a (mon.0) 358 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:42:20.874 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:20 smithi089 bash[21793]: audit 2024-07-15T06:42:19.705756+0000 mon.a (mon.0) 359 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:42:20.874 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:20 smithi089 bash[21793]: audit 2024-07-15T06:42:19.712584+0000 mon.a (mon.0) 360 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:42:20.874 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:20 smithi089 bash[31372]: audit 2024-07-15T06:42:19.582416+0000 mon.a (mon.0) 354 : audit [INF] from='osd.2 [v2:172.21.15.89:6818/3236432318,v1:172.21.15.89:6819/3236432318]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]': finished 2024-07-15T06:42:20.874 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:20 smithi089 bash[31372]: cluster 2024-07-15T06:42:19.582584+0000 mon.a (mon.0) 355 : cluster [DBG] osdmap e17: 3 total, 2 up, 3 in 2024-07-15T06:42:20.874 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:20 smithi089 bash[31372]: audit 2024-07-15T06:42:19.583022+0000 mon.a (mon.0) 356 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-07-15T06:42:20.874 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:20 smithi089 bash[31372]: audit 2024-07-15T06:42:19.589606+0000 mon.a (mon.0) 357 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-07-15T06:42:20.875 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:20 smithi089 bash[31372]: audit 2024-07-15T06:42:19.699585+0000 mon.a (mon.0) 358 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:42:20.875 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:20 smithi089 bash[31372]: audit 2024-07-15T06:42:19.705756+0000 mon.a (mon.0) 359 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:42:20.875 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:20 smithi089 bash[31372]: audit 2024-07-15T06:42:19.712584+0000 mon.a (mon.0) 360 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:42:21.059 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:20 smithi134 bash[24068]: audit 2024-07-15T06:42:19.582416+0000 mon.a (mon.0) 354 : audit [INF] from='osd.2 [v2:172.21.15.89:6818/3236432318,v1:172.21.15.89:6819/3236432318]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]': finished 2024-07-15T06:42:21.059 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:20 smithi134 bash[24068]: cluster 2024-07-15T06:42:19.582584+0000 mon.a (mon.0) 355 : cluster [DBG] osdmap e17: 3 total, 2 up, 3 in 2024-07-15T06:42:21.059 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:20 smithi134 bash[24068]: audit 2024-07-15T06:42:19.583022+0000 mon.a (mon.0) 356 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-07-15T06:42:21.059 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:20 smithi134 bash[24068]: audit 2024-07-15T06:42:19.589606+0000 mon.a (mon.0) 357 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-07-15T06:42:21.060 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:20 smithi134 bash[24068]: audit 2024-07-15T06:42:19.699585+0000 mon.a (mon.0) 358 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:42:21.060 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:20 smithi134 bash[24068]: audit 2024-07-15T06:42:19.705756+0000 mon.a (mon.0) 359 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:42:21.060 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:20 smithi134 bash[24068]: audit 2024-07-15T06:42:19.712584+0000 mon.a (mon.0) 360 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:42:21.873 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:21 smithi089 bash[21793]: cluster 2024-07-15T06:42:18.730937+0000 osd.2 (osd.2) 1 : cluster [DBG] purged_snaps scrub starts 2024-07-15T06:42:21.873 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:21 smithi089 bash[21793]: cluster 2024-07-15T06:42:18.731013+0000 osd.2 (osd.2) 2 : cluster [DBG] purged_snaps scrub ok 2024-07-15T06:42:21.874 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:21 smithi089 bash[21793]: cluster 2024-07-15T06:42:20.581215+0000 mon.a (mon.0) 361 : cluster [INF] osd.2 [v2:172.21.15.89:6818/3236432318,v1:172.21.15.89:6819/3236432318] boot 2024-07-15T06:42:21.874 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:21 smithi089 bash[21793]: cluster 2024-07-15T06:42:20.581296+0000 mon.a (mon.0) 362 : cluster [DBG] osdmap e18: 3 total, 3 up, 3 in 2024-07-15T06:42:21.874 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:21 smithi089 bash[21793]: audit 2024-07-15T06:42:20.581850+0000 mon.a (mon.0) 363 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-07-15T06:42:21.874 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:21 smithi089 bash[21793]: cluster 2024-07-15T06:42:20.904220+0000 mgr.y (mgr.14150) 168 : cluster [DBG] pgmap v135: 0 pgs: ; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-07-15T06:42:21.874 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:21 smithi089 bash[21793]: audit 2024-07-15T06:42:21.006353+0000 mon.a (mon.0) 364 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-07-15T06:42:21.874 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:21 smithi089 bash[31372]: cluster 2024-07-15T06:42:18.730937+0000 osd.2 (osd.2) 1 : cluster [DBG] purged_snaps scrub starts 2024-07-15T06:42:21.874 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:21 smithi089 bash[31372]: cluster 2024-07-15T06:42:18.731013+0000 osd.2 (osd.2) 2 : cluster [DBG] purged_snaps scrub ok 2024-07-15T06:42:21.874 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:21 smithi089 bash[31372]: cluster 2024-07-15T06:42:20.581215+0000 mon.a (mon.0) 361 : cluster [INF] osd.2 [v2:172.21.15.89:6818/3236432318,v1:172.21.15.89:6819/3236432318] boot 2024-07-15T06:42:21.874 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:21 smithi089 bash[31372]: cluster 2024-07-15T06:42:20.581296+0000 mon.a (mon.0) 362 : cluster [DBG] osdmap e18: 3 total, 3 up, 3 in 2024-07-15T06:42:21.874 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:21 smithi089 bash[31372]: audit 2024-07-15T06:42:20.581850+0000 mon.a (mon.0) 363 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-07-15T06:42:21.875 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:21 smithi089 bash[31372]: cluster 2024-07-15T06:42:20.904220+0000 mgr.y (mgr.14150) 168 : cluster [DBG] pgmap v135: 0 pgs: ; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-07-15T06:42:21.875 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:21 smithi089 bash[31372]: audit 2024-07-15T06:42:21.006353+0000 mon.a (mon.0) 364 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-07-15T06:42:22.059 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:21 smithi134 bash[24068]: cluster 2024-07-15T06:42:18.730937+0000 osd.2 (osd.2) 1 : cluster [DBG] purged_snaps scrub starts 2024-07-15T06:42:22.059 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:21 smithi134 bash[24068]: cluster 2024-07-15T06:42:18.731013+0000 osd.2 (osd.2) 2 : cluster [DBG] purged_snaps scrub ok 2024-07-15T06:42:22.059 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:21 smithi134 bash[24068]: cluster 2024-07-15T06:42:20.581215+0000 mon.a (mon.0) 361 : cluster [INF] osd.2 [v2:172.21.15.89:6818/3236432318,v1:172.21.15.89:6819/3236432318] boot 2024-07-15T06:42:22.059 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:21 smithi134 bash[24068]: cluster 2024-07-15T06:42:20.581296+0000 mon.a (mon.0) 362 : cluster [DBG] osdmap e18: 3 total, 3 up, 3 in 2024-07-15T06:42:22.059 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:21 smithi134 bash[24068]: audit 2024-07-15T06:42:20.581850+0000 mon.a (mon.0) 363 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-07-15T06:42:22.060 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:21 smithi134 bash[24068]: cluster 2024-07-15T06:42:20.904220+0000 mgr.y (mgr.14150) 168 : cluster [DBG] pgmap v135: 0 pgs: ; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-07-15T06:42:22.060 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:21 smithi134 bash[24068]: audit 2024-07-15T06:42:21.006353+0000 mon.a (mon.0) 364 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-07-15T06:42:22.873 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:22 smithi089 bash[21793]: audit 2024-07-15T06:42:21.596785+0000 mon.a (mon.0) 365 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-07-15T06:42:22.873 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:22 smithi089 bash[21793]: cluster 2024-07-15T06:42:21.596978+0000 mon.a (mon.0) 366 : cluster [DBG] osdmap e19: 3 total, 3 up, 3 in 2024-07-15T06:42:22.874 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:22 smithi089 bash[21793]: audit 2024-07-15T06:42:21.605057+0000 mon.a (mon.0) 367 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-07-15T06:42:22.874 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:22 smithi089 bash[21793]: audit 2024-07-15T06:42:22.593422+0000 mon.a (mon.0) 368 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-07-15T06:42:22.874 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:22 smithi089 bash[21793]: cluster 2024-07-15T06:42:22.593572+0000 mon.a (mon.0) 369 : cluster [DBG] osdmap e20: 3 total, 3 up, 3 in 2024-07-15T06:42:22.874 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:22 smithi089 bash[31372]: audit 2024-07-15T06:42:21.596785+0000 mon.a (mon.0) 365 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-07-15T06:42:22.874 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:22 smithi089 bash[31372]: cluster 2024-07-15T06:42:21.596978+0000 mon.a (mon.0) 366 : cluster [DBG] osdmap e19: 3 total, 3 up, 3 in 2024-07-15T06:42:22.874 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:22 smithi089 bash[31372]: audit 2024-07-15T06:42:21.605057+0000 mon.a (mon.0) 367 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-07-15T06:42:22.874 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:22 smithi089 bash[31372]: audit 2024-07-15T06:42:22.593422+0000 mon.a (mon.0) 368 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-07-15T06:42:22.874 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:22 smithi089 bash[31372]: cluster 2024-07-15T06:42:22.593572+0000 mon.a (mon.0) 369 : cluster [DBG] osdmap e20: 3 total, 3 up, 3 in 2024-07-15T06:42:23.059 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:22 smithi134 bash[24068]: audit 2024-07-15T06:42:21.596785+0000 mon.a (mon.0) 365 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-07-15T06:42:23.059 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:22 smithi134 bash[24068]: cluster 2024-07-15T06:42:21.596978+0000 mon.a (mon.0) 366 : cluster [DBG] osdmap e19: 3 total, 3 up, 3 in 2024-07-15T06:42:23.059 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:22 smithi134 bash[24068]: audit 2024-07-15T06:42:21.605057+0000 mon.a (mon.0) 367 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-07-15T06:42:23.059 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:22 smithi134 bash[24068]: audit 2024-07-15T06:42:22.593422+0000 mon.a (mon.0) 368 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-07-15T06:42:23.060 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:22 smithi134 bash[24068]: cluster 2024-07-15T06:42:22.593572+0000 mon.a (mon.0) 369 : cluster [DBG] osdmap e20: 3 total, 3 up, 3 in 2024-07-15T06:42:23.873 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:23 smithi089 bash[21793]: cluster 2024-07-15T06:42:22.904724+0000 mgr.y (mgr.14150) 169 : cluster [DBG] pgmap v138: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-07-15T06:42:23.873 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:23 smithi089 bash[31372]: cluster 2024-07-15T06:42:22.904724+0000 mgr.y (mgr.14150) 169 : cluster [DBG] pgmap v138: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-07-15T06:42:24.062 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:23 smithi134 bash[24068]: cluster 2024-07-15T06:42:22.904724+0000 mgr.y (mgr.14150) 169 : cluster [DBG] pgmap v138: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-07-15T06:42:24.480 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T06:42:24.873 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:24 smithi089 bash[21793]: cluster 2024-07-15T06:42:23.623169+0000 mon.a (mon.0) 370 : cluster [DBG] osdmap e21: 3 total, 3 up, 3 in 2024-07-15T06:42:24.873 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:24 smithi089 bash[21793]: cluster 2024-07-15T06:42:23.623957+0000 mon.a (mon.0) 371 : cluster [DBG] mgrmap e15: y(active, since 4m), standbys: x 2024-07-15T06:42:24.873 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:24 smithi089 bash[21793]: audit 2024-07-15T06:42:23.657019+0000 mon.a (mon.0) 372 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:42:24.873 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:24 smithi089 bash[31372]: cluster 2024-07-15T06:42:23.623169+0000 mon.a (mon.0) 370 : cluster [DBG] osdmap e21: 3 total, 3 up, 3 in 2024-07-15T06:42:24.873 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:24 smithi089 bash[31372]: cluster 2024-07-15T06:42:23.623957+0000 mon.a (mon.0) 371 : cluster [DBG] mgrmap e15: y(active, since 4m), standbys: x 2024-07-15T06:42:24.874 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:24 smithi089 bash[31372]: audit 2024-07-15T06:42:23.657019+0000 mon.a (mon.0) 372 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:42:25.059 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:24 smithi134 bash[24068]: cluster 2024-07-15T06:42:23.623169+0000 mon.a (mon.0) 370 : cluster [DBG] osdmap e21: 3 total, 3 up, 3 in 2024-07-15T06:42:25.059 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:24 smithi134 bash[24068]: cluster 2024-07-15T06:42:23.623957+0000 mon.a (mon.0) 371 : cluster [DBG] mgrmap e15: y(active, since 4m), standbys: x 2024-07-15T06:42:25.059 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:24 smithi134 bash[24068]: audit 2024-07-15T06:42:23.657019+0000 mon.a (mon.0) 372 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:42:27.373 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:27 smithi089 bash[21793]: cluster 2024-07-15T06:42:24.905238+0000 mgr.y (mgr.14150) 170 : cluster [DBG] pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-07-15T06:42:27.373 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:27 smithi089 bash[21793]: audit 2024-07-15T06:42:25.385479+0000 mon.a (mon.0) 373 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-07-15T06:42:27.373 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:27 smithi089 bash[21793]: audit 2024-07-15T06:42:26.270534+0000 mon.a (mon.0) 374 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-07-15T06:42:27.373 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:27 smithi089 bash[21793]: audit 2024-07-15T06:42:26.272047+0000 mon.a (mon.0) 375 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-07-15T06:42:27.373 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:27 smithi089 bash[21793]: audit 2024-07-15T06:42:26.274580+0000 mon.a (mon.0) 376 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-15T06:42:27.374 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:27 smithi089 bash[21793]: audit 2024-07-15T06:42:26.274761+0000 mon.a (mon.0) 377 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-15T06:42:27.374 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:27 smithi089 bash[21793]: audit 2024-07-15T06:42:26.277300+0000 mon.a (mon.0) 378 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-07-15T06:42:27.374 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:27 smithi089 bash[21793]: audit 2024-07-15T06:42:26.277415+0000 mon.a (mon.0) 379 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-15T06:42:27.374 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:27 smithi089 bash[21793]: audit 2024-07-15T06:42:26.277596+0000 mon.a (mon.0) 380 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-15T06:42:27.374 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:27 smithi089 bash[31372]: cluster 2024-07-15T06:42:24.905238+0000 mgr.y (mgr.14150) 170 : cluster [DBG] pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-07-15T06:42:27.374 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:27 smithi089 bash[31372]: audit 2024-07-15T06:42:25.385479+0000 mon.a (mon.0) 373 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-07-15T06:42:27.374 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:27 smithi089 bash[31372]: audit 2024-07-15T06:42:26.270534+0000 mon.a (mon.0) 374 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-07-15T06:42:27.374 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:27 smithi089 bash[31372]: audit 2024-07-15T06:42:26.272047+0000 mon.a (mon.0) 375 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-07-15T06:42:27.374 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:27 smithi089 bash[31372]: audit 2024-07-15T06:42:26.274580+0000 mon.a (mon.0) 376 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-15T06:42:27.374 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:27 smithi089 bash[31372]: audit 2024-07-15T06:42:26.274761+0000 mon.a (mon.0) 377 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-15T06:42:27.374 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:27 smithi089 bash[31372]: audit 2024-07-15T06:42:26.277300+0000 mon.a (mon.0) 378 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-07-15T06:42:27.375 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:27 smithi089 bash[31372]: audit 2024-07-15T06:42:26.277415+0000 mon.a (mon.0) 379 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-15T06:42:27.375 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:27 smithi089 bash[31372]: audit 2024-07-15T06:42:26.277596+0000 mon.a (mon.0) 380 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-15T06:42:27.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:27 smithi134 bash[24068]: cluster 2024-07-15T06:42:24.905238+0000 mgr.y (mgr.14150) 170 : cluster [DBG] pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-07-15T06:42:27.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:27 smithi134 bash[24068]: audit 2024-07-15T06:42:25.385479+0000 mon.a (mon.0) 373 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-07-15T06:42:27.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:27 smithi134 bash[24068]: audit 2024-07-15T06:42:26.270534+0000 mon.a (mon.0) 374 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-07-15T06:42:27.560 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:27 smithi134 bash[24068]: audit 2024-07-15T06:42:26.272047+0000 mon.a (mon.0) 375 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-07-15T06:42:27.560 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:27 smithi134 bash[24068]: audit 2024-07-15T06:42:26.274580+0000 mon.a (mon.0) 376 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-15T06:42:27.560 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:27 smithi134 bash[24068]: audit 2024-07-15T06:42:26.274761+0000 mon.a (mon.0) 377 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-15T06:42:27.560 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:27 smithi134 bash[24068]: audit 2024-07-15T06:42:26.277300+0000 mon.a (mon.0) 378 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-07-15T06:42:27.560 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:27 smithi134 bash[24068]: audit 2024-07-15T06:42:26.277415+0000 mon.a (mon.0) 379 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-15T06:42:27.560 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:27 smithi134 bash[24068]: audit 2024-07-15T06:42:26.277596+0000 mon.a (mon.0) 380 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-15T06:42:28.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:28 smithi134 bash[24068]: audit 2024-07-15T06:42:26.276017+0000 mon.b (mon.1) 7 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-07-15T06:42:28.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:28 smithi134 bash[24068]: cluster 2024-07-15T06:42:26.905735+0000 mgr.y (mgr.14150) 171 : cluster [DBG] pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-07-15T06:42:28.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:28 smithi134 bash[24068]: audit 2024-07-15T06:42:27.178405+0000 mon.b (mon.1) 8 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-07-15T06:42:28.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:28 smithi134 bash[24068]: audit 2024-07-15T06:42:27.227231+0000 mon.a (mon.0) 381 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:42:28.560 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:28 smithi134 bash[24068]: audit 2024-07-15T06:42:27.229128+0000 mon.c (mon.2) 8 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-07-15T06:42:28.560 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:28 smithi134 bash[24068]: audit 2024-07-15T06:42:27.229416+0000 mon.a (mon.0) 382 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-07-15T06:42:28.560 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:28 smithi134 bash[24068]: audit 2024-07-15T06:42:27.229747+0000 mon.a (mon.0) 383 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-15T06:42:28.560 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:28 smithi134 bash[24068]: audit 2024-07-15T06:42:27.230071+0000 mon.a (mon.0) 384 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-15T06:42:28.560 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:28 smithi134 bash[24068]: audit 2024-07-15T06:42:28.143737+0000 mon.c (mon.2) 9 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-07-15T06:42:28.623 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:28 smithi089 bash[21793]: audit 2024-07-15T06:42:26.276017+0000 mon.b (mon.1) 7 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-07-15T06:42:28.623 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:28 smithi089 bash[21793]: cluster 2024-07-15T06:42:26.905735+0000 mgr.y (mgr.14150) 171 : cluster [DBG] pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-07-15T06:42:28.623 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:28 smithi089 bash[21793]: audit 2024-07-15T06:42:27.178405+0000 mon.b (mon.1) 8 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-07-15T06:42:28.624 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:28 smithi089 bash[21793]: audit 2024-07-15T06:42:27.227231+0000 mon.a (mon.0) 381 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:42:28.624 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:28 smithi089 bash[21793]: audit 2024-07-15T06:42:27.229128+0000 mon.c (mon.2) 8 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-07-15T06:42:28.624 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:28 smithi089 bash[21793]: audit 2024-07-15T06:42:27.229416+0000 mon.a (mon.0) 382 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-07-15T06:42:28.624 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:28 smithi089 bash[21793]: audit 2024-07-15T06:42:27.229747+0000 mon.a (mon.0) 383 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-15T06:42:28.624 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:28 smithi089 bash[21793]: audit 2024-07-15T06:42:27.230071+0000 mon.a (mon.0) 384 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-15T06:42:28.624 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:28 smithi089 bash[21793]: audit 2024-07-15T06:42:28.143737+0000 mon.c (mon.2) 9 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-07-15T06:42:28.624 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:28 smithi089 bash[31372]: audit 2024-07-15T06:42:26.276017+0000 mon.b (mon.1) 7 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-07-15T06:42:28.624 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:28 smithi089 bash[31372]: cluster 2024-07-15T06:42:26.905735+0000 mgr.y (mgr.14150) 171 : cluster [DBG] pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-07-15T06:42:28.624 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:28 smithi089 bash[31372]: audit 2024-07-15T06:42:27.178405+0000 mon.b (mon.1) 8 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-07-15T06:42:28.624 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:28 smithi089 bash[31372]: audit 2024-07-15T06:42:27.227231+0000 mon.a (mon.0) 381 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:42:28.624 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:28 smithi089 bash[31372]: audit 2024-07-15T06:42:27.229128+0000 mon.c (mon.2) 8 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-07-15T06:42:28.625 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:28 smithi089 bash[31372]: audit 2024-07-15T06:42:27.229416+0000 mon.a (mon.0) 382 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-07-15T06:42:28.625 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:28 smithi089 bash[31372]: audit 2024-07-15T06:42:27.229747+0000 mon.a (mon.0) 383 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-15T06:42:28.625 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:28 smithi089 bash[31372]: audit 2024-07-15T06:42:27.230071+0000 mon.a (mon.0) 384 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-15T06:42:28.625 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:28 smithi089 bash[31372]: audit 2024-07-15T06:42:28.143737+0000 mon.c (mon.2) 9 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-07-15T06:42:29.358 INFO:teuthology.orchestra.run.smithi089.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-07-15T06:42:29.559 INFO:teuthology.orchestra.run.smithi089.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-07-15T06:42:29.576 INFO:teuthology.orchestra.run.smithi089.stderr: stderr: 10+0 records in 2024-07-15T06:42:29.576 INFO:teuthology.orchestra.run.smithi089.stderr:10+0 records out 2024-07-15T06:42:29.576 INFO:teuthology.orchestra.run.smithi089.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0142968 s, 733 MB/s 2024-07-15T06:42:29.577 INFO:teuthology.orchestra.run.smithi089.stderr:--> Zapping successful for: 2024-07-15T06:42:30.436 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph orch daemon add osd smithi089:vg_nvme/lv_1 2024-07-15T06:42:30.446 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:30 smithi089 bash[21793]: cluster 2024-07-15T06:42:28.906261+0000 mgr.y (mgr.14150) 172 : cluster [DBG] pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-07-15T06:42:30.446 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:30 smithi089 bash[31372]: cluster 2024-07-15T06:42:28.906261+0000 mgr.y (mgr.14150) 172 : cluster [DBG] pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-07-15T06:42:30.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:30 smithi134 bash[24068]: cluster 2024-07-15T06:42:28.906261+0000 mgr.y (mgr.14150) 172 : cluster [DBG] pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-07-15T06:42:32.123 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:31 smithi089 bash[21793]: cephadm 2024-07-15T06:42:30.805882+0000 mgr.y (mgr.14150) 173 : cephadm [INF] Detected new or changed devices on smithi089 2024-07-15T06:42:32.123 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:31 smithi089 bash[21793]: audit 2024-07-15T06:42:30.814358+0000 mon.a (mon.0) 385 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:42:32.124 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:31 smithi089 bash[21793]: audit 2024-07-15T06:42:30.822968+0000 mon.a (mon.0) 386 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:42:32.124 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:31 smithi089 bash[21793]: audit 2024-07-15T06:42:30.825158+0000 mon.a (mon.0) 387 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:42:32.124 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:31 smithi089 bash[21793]: audit 2024-07-15T06:42:30.826831+0000 mon.a (mon.0) 388 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:42:32.124 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:31 smithi089 bash[21793]: audit 2024-07-15T06:42:30.828391+0000 mon.a (mon.0) 389 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:42:32.124 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:31 smithi089 bash[21793]: cephadm 2024-07-15T06:42:30.829584+0000 mgr.y (mgr.14150) 174 : cephadm [INF] Adjusting osd_memory_target on smithi089 to 5413M 2024-07-15T06:42:32.124 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:31 smithi089 bash[21793]: audit 2024-07-15T06:42:30.836830+0000 mon.a (mon.0) 390 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:42:32.124 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:31 smithi089 bash[21793]: audit 2024-07-15T06:42:30.839810+0000 mon.a (mon.0) 391 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:42:32.124 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:31 smithi089 bash[21793]: audit 2024-07-15T06:42:30.841458+0000 mon.a (mon.0) 392 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:42:32.124 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:31 smithi089 bash[21793]: audit 2024-07-15T06:42:30.850391+0000 mon.a (mon.0) 393 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:42:32.124 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:31 smithi089 bash[21793]: cluster 2024-07-15T06:42:30.906839+0000 mgr.y (mgr.14150) 175 : cluster [DBG] pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-07-15T06:42:32.124 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:31 smithi089 bash[31372]: cephadm 2024-07-15T06:42:30.805882+0000 mgr.y (mgr.14150) 173 : cephadm [INF] Detected new or changed devices on smithi089 2024-07-15T06:42:32.125 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:31 smithi089 bash[31372]: audit 2024-07-15T06:42:30.814358+0000 mon.a (mon.0) 385 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:42:32.125 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:31 smithi089 bash[31372]: audit 2024-07-15T06:42:30.822968+0000 mon.a (mon.0) 386 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:42:32.125 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:31 smithi089 bash[31372]: audit 2024-07-15T06:42:30.825158+0000 mon.a (mon.0) 387 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:42:32.125 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:31 smithi089 bash[31372]: audit 2024-07-15T06:42:30.826831+0000 mon.a (mon.0) 388 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:42:32.125 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:31 smithi089 bash[31372]: audit 2024-07-15T06:42:30.828391+0000 mon.a (mon.0) 389 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:42:32.125 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:31 smithi089 bash[31372]: cephadm 2024-07-15T06:42:30.829584+0000 mgr.y (mgr.14150) 174 : cephadm [INF] Adjusting osd_memory_target on smithi089 to 5413M 2024-07-15T06:42:32.125 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:31 smithi089 bash[31372]: audit 2024-07-15T06:42:30.836830+0000 mon.a (mon.0) 390 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:42:32.125 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:31 smithi089 bash[31372]: audit 2024-07-15T06:42:30.839810+0000 mon.a (mon.0) 391 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:42:32.125 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:31 smithi089 bash[31372]: audit 2024-07-15T06:42:30.841458+0000 mon.a (mon.0) 392 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:42:32.125 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:31 smithi089 bash[31372]: audit 2024-07-15T06:42:30.850391+0000 mon.a (mon.0) 393 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:42:32.125 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:31 smithi089 bash[31372]: cluster 2024-07-15T06:42:30.906839+0000 mgr.y (mgr.14150) 175 : cluster [DBG] pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-07-15T06:42:32.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:31 smithi134 bash[24068]: cephadm 2024-07-15T06:42:30.805882+0000 mgr.y (mgr.14150) 173 : cephadm [INF] Detected new or changed devices on smithi089 2024-07-15T06:42:32.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:31 smithi134 bash[24068]: audit 2024-07-15T06:42:30.814358+0000 mon.a (mon.0) 385 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:42:32.310 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:31 smithi134 bash[24068]: audit 2024-07-15T06:42:30.822968+0000 mon.a (mon.0) 386 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:42:32.310 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:31 smithi134 bash[24068]: audit 2024-07-15T06:42:30.825158+0000 mon.a (mon.0) 387 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:42:32.310 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:31 smithi134 bash[24068]: audit 2024-07-15T06:42:30.826831+0000 mon.a (mon.0) 388 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:42:32.310 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:31 smithi134 bash[24068]: audit 2024-07-15T06:42:30.828391+0000 mon.a (mon.0) 389 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:42:32.310 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:31 smithi134 bash[24068]: cephadm 2024-07-15T06:42:30.829584+0000 mgr.y (mgr.14150) 174 : cephadm [INF] Adjusting osd_memory_target on smithi089 to 5413M 2024-07-15T06:42:32.310 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:31 smithi134 bash[24068]: audit 2024-07-15T06:42:30.836830+0000 mon.a (mon.0) 390 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:42:32.310 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:31 smithi134 bash[24068]: audit 2024-07-15T06:42:30.839810+0000 mon.a (mon.0) 391 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:42:32.310 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:31 smithi134 bash[24068]: audit 2024-07-15T06:42:30.841458+0000 mon.a (mon.0) 392 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:42:32.310 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:31 smithi134 bash[24068]: audit 2024-07-15T06:42:30.850391+0000 mon.a (mon.0) 393 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:42:32.310 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:31 smithi134 bash[24068]: cluster 2024-07-15T06:42:30.906839+0000 mgr.y (mgr.14150) 175 : cluster [DBG] pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-07-15T06:42:34.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:33 smithi134 bash[24068]: cluster 2024-07-15T06:42:32.907401+0000 mgr.y (mgr.14150) 176 : cluster [DBG] pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-07-15T06:42:34.373 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:33 smithi089 bash[21793]: cluster 2024-07-15T06:42:32.907401+0000 mgr.y (mgr.14150) 176 : cluster [DBG] pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-07-15T06:42:34.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:33 smithi089 bash[31372]: cluster 2024-07-15T06:42:32.907401+0000 mgr.y (mgr.14150) 176 : cluster [DBG] pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-07-15T06:42:35.187 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T06:42:36.234 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:35 smithi089 bash[31372]: cluster 2024-07-15T06:42:34.907946+0000 mgr.y (mgr.14150) 177 : cluster [DBG] pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-07-15T06:42:36.234 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:35 smithi089 bash[21793]: cluster 2024-07-15T06:42:34.907946+0000 mgr.y (mgr.14150) 177 : cluster [DBG] pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-07-15T06:42:36.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:35 smithi134 bash[24068]: cluster 2024-07-15T06:42:34.907946+0000 mgr.y (mgr.14150) 177 : cluster [DBG] pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-07-15T06:42:38.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:37 smithi134 bash[24068]: cluster 2024-07-15T06:42:36.908494+0000 mgr.y (mgr.14150) 178 : cluster [DBG] pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-07-15T06:42:38.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:37 smithi089 bash[21793]: cluster 2024-07-15T06:42:36.908494+0000 mgr.y (mgr.14150) 178 : cluster [DBG] pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-07-15T06:42:38.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:37 smithi089 bash[31372]: cluster 2024-07-15T06:42:36.908494+0000 mgr.y (mgr.14150) 178 : cluster [DBG] pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-07-15T06:42:39.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:38 smithi134 bash[24068]: audit 2024-07-15T06:42:38.274365+0000 mgr.y (mgr.14150) 179 : audit [DBG] from='client.14298 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi089:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T06:42:39.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:38 smithi134 bash[24068]: audit 2024-07-15T06:42:38.278776+0000 mon.a (mon.0) 394 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-07-15T06:42:39.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:38 smithi134 bash[24068]: audit 2024-07-15T06:42:38.284040+0000 mon.a (mon.0) 395 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-07-15T06:42:39.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:38 smithi134 bash[24068]: audit 2024-07-15T06:42:38.285461+0000 mon.a (mon.0) 396 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:42:39.373 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:38 smithi089 bash[21793]: audit 2024-07-15T06:42:38.274365+0000 mgr.y (mgr.14150) 179 : audit [DBG] from='client.14298 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi089:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T06:42:39.373 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:38 smithi089 bash[21793]: audit 2024-07-15T06:42:38.278776+0000 mon.a (mon.0) 394 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-07-15T06:42:39.373 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:38 smithi089 bash[21793]: audit 2024-07-15T06:42:38.284040+0000 mon.a (mon.0) 395 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-07-15T06:42:39.373 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:38 smithi089 bash[21793]: audit 2024-07-15T06:42:38.285461+0000 mon.a (mon.0) 396 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:42:39.374 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:38 smithi089 bash[31372]: audit 2024-07-15T06:42:38.274365+0000 mgr.y (mgr.14150) 179 : audit [DBG] from='client.14298 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi089:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T06:42:39.374 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:38 smithi089 bash[31372]: audit 2024-07-15T06:42:38.278776+0000 mon.a (mon.0) 394 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-07-15T06:42:39.374 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:38 smithi089 bash[31372]: audit 2024-07-15T06:42:38.284040+0000 mon.a (mon.0) 395 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-07-15T06:42:39.374 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:38 smithi089 bash[31372]: audit 2024-07-15T06:42:38.285461+0000 mon.a (mon.0) 396 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:42:40.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:39 smithi134 bash[24068]: cluster 2024-07-15T06:42:38.909015+0000 mgr.y (mgr.14150) 180 : cluster [DBG] pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-07-15T06:42:40.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:39 smithi089 bash[31372]: cluster 2024-07-15T06:42:38.909015+0000 mgr.y (mgr.14150) 180 : cluster [DBG] pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-07-15T06:42:40.373 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:39 smithi089 bash[21793]: cluster 2024-07-15T06:42:38.909015+0000 mgr.y (mgr.14150) 180 : cluster [DBG] pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-07-15T06:42:42.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:41 smithi134 bash[24068]: cluster 2024-07-15T06:42:40.909617+0000 mgr.y (mgr.14150) 181 : cluster [DBG] pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-07-15T06:42:42.372 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:41 smithi089 bash[31372]: cluster 2024-07-15T06:42:40.909617+0000 mgr.y (mgr.14150) 181 : cluster [DBG] pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-07-15T06:42:42.373 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:41 smithi089 bash[21793]: cluster 2024-07-15T06:42:40.909617+0000 mgr.y (mgr.14150) 181 : cluster [DBG] pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-07-15T06:42:44.250 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:43 smithi089 bash[31372]: cluster 2024-07-15T06:42:42.910176+0000 mgr.y (mgr.14150) 182 : cluster [DBG] pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-07-15T06:42:44.250 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:43 smithi089 bash[21793]: cluster 2024-07-15T06:42:42.910176+0000 mgr.y (mgr.14150) 182 : cluster [DBG] pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-07-15T06:42:44.308 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:43 smithi134 bash[24068]: cluster 2024-07-15T06:42:42.910176+0000 mgr.y (mgr.14150) 182 : cluster [DBG] pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-07-15T06:42:46.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:45 smithi134 bash[24068]: cluster 2024-07-15T06:42:44.910790+0000 mgr.y (mgr.14150) 183 : cluster [DBG] pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-07-15T06:42:46.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:45 smithi089 bash[21793]: cluster 2024-07-15T06:42:44.910790+0000 mgr.y (mgr.14150) 183 : cluster [DBG] pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-07-15T06:42:46.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:45 smithi089 bash[31372]: cluster 2024-07-15T06:42:44.910790+0000 mgr.y (mgr.14150) 183 : cluster [DBG] pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-07-15T06:42:47.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:46 smithi134 bash[24068]: audit 2024-07-15T06:42:46.708676+0000 mon.c (mon.2) 10 : audit [INF] from='client.? 172.21.15.89:0/3729683081' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8af52bbd-4bc9-4b13-b525-26c92e69c29f"}]: dispatch 2024-07-15T06:42:47.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:46 smithi134 bash[24068]: audit 2024-07-15T06:42:46.709381+0000 mon.a (mon.0) 397 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8af52bbd-4bc9-4b13-b525-26c92e69c29f"}]: dispatch 2024-07-15T06:42:47.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:46 smithi134 bash[24068]: audit 2024-07-15T06:42:46.719231+0000 mon.a (mon.0) 398 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "8af52bbd-4bc9-4b13-b525-26c92e69c29f"}]': finished 2024-07-15T06:42:47.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:46 smithi134 bash[24068]: cluster 2024-07-15T06:42:46.719447+0000 mon.a (mon.0) 399 : cluster [DBG] osdmap e22: 4 total, 3 up, 4 in 2024-07-15T06:42:47.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:46 smithi134 bash[24068]: audit 2024-07-15T06:42:46.719651+0000 mon.a (mon.0) 400 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-07-15T06:42:47.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:46 smithi089 bash[21793]: audit 2024-07-15T06:42:46.708676+0000 mon.c (mon.2) 10 : audit [INF] from='client.? 172.21.15.89:0/3729683081' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8af52bbd-4bc9-4b13-b525-26c92e69c29f"}]: dispatch 2024-07-15T06:42:47.373 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:46 smithi089 bash[21793]: audit 2024-07-15T06:42:46.709381+0000 mon.a (mon.0) 397 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8af52bbd-4bc9-4b13-b525-26c92e69c29f"}]: dispatch 2024-07-15T06:42:47.373 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:46 smithi089 bash[21793]: audit 2024-07-15T06:42:46.719231+0000 mon.a (mon.0) 398 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "8af52bbd-4bc9-4b13-b525-26c92e69c29f"}]': finished 2024-07-15T06:42:47.373 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:46 smithi089 bash[21793]: cluster 2024-07-15T06:42:46.719447+0000 mon.a (mon.0) 399 : cluster [DBG] osdmap e22: 4 total, 3 up, 4 in 2024-07-15T06:42:47.373 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:46 smithi089 bash[21793]: audit 2024-07-15T06:42:46.719651+0000 mon.a (mon.0) 400 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-07-15T06:42:47.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:46 smithi089 bash[31372]: audit 2024-07-15T06:42:46.708676+0000 mon.c (mon.2) 10 : audit [INF] from='client.? 172.21.15.89:0/3729683081' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8af52bbd-4bc9-4b13-b525-26c92e69c29f"}]: dispatch 2024-07-15T06:42:47.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:46 smithi089 bash[31372]: audit 2024-07-15T06:42:46.709381+0000 mon.a (mon.0) 397 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8af52bbd-4bc9-4b13-b525-26c92e69c29f"}]: dispatch 2024-07-15T06:42:47.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:46 smithi089 bash[31372]: audit 2024-07-15T06:42:46.719231+0000 mon.a (mon.0) 398 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "8af52bbd-4bc9-4b13-b525-26c92e69c29f"}]': finished 2024-07-15T06:42:47.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:46 smithi089 bash[31372]: cluster 2024-07-15T06:42:46.719447+0000 mon.a (mon.0) 399 : cluster [DBG] osdmap e22: 4 total, 3 up, 4 in 2024-07-15T06:42:47.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:46 smithi089 bash[31372]: audit 2024-07-15T06:42:46.719651+0000 mon.a (mon.0) 400 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-07-15T06:42:48.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:47 smithi134 bash[24068]: cluster 2024-07-15T06:42:46.911403+0000 mgr.y (mgr.14150) 184 : cluster [DBG] pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-07-15T06:42:48.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:47 smithi134 bash[24068]: audit 2024-07-15T06:42:47.346346+0000 mon.c (mon.2) 11 : audit [DBG] from='client.? 172.21.15.89:0/667105641' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-07-15T06:42:48.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:47 smithi089 bash[31372]: cluster 2024-07-15T06:42:46.911403+0000 mgr.y (mgr.14150) 184 : cluster [DBG] pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-07-15T06:42:48.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:47 smithi089 bash[31372]: audit 2024-07-15T06:42:47.346346+0000 mon.c (mon.2) 11 : audit [DBG] from='client.? 172.21.15.89:0/667105641' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-07-15T06:42:48.373 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:47 smithi089 bash[21793]: cluster 2024-07-15T06:42:46.911403+0000 mgr.y (mgr.14150) 184 : cluster [DBG] pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-07-15T06:42:48.373 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:47 smithi089 bash[21793]: audit 2024-07-15T06:42:47.346346+0000 mon.c (mon.2) 11 : audit [DBG] from='client.? 172.21.15.89:0/667105641' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-07-15T06:42:50.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:50 smithi089 bash[31372]: cluster 2024-07-15T06:42:48.912065+0000 mgr.y (mgr.14150) 185 : cluster [DBG] pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-07-15T06:42:50.123 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:49 smithi089 bash[21793]: cluster 2024-07-15T06:42:48.912065+0000 mgr.y (mgr.14150) 185 : cluster [DBG] pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-07-15T06:42:50.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:49 smithi134 bash[24068]: cluster 2024-07-15T06:42:48.912065+0000 mgr.y (mgr.14150) 185 : cluster [DBG] pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-07-15T06:42:52.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:52 smithi134 bash[24068]: cluster 2024-07-15T06:42:50.912746+0000 mgr.y (mgr.14150) 186 : cluster [DBG] pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-07-15T06:42:52.372 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:52 smithi089 bash[31372]: cluster 2024-07-15T06:42:50.912746+0000 mgr.y (mgr.14150) 186 : cluster [DBG] pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-07-15T06:42:52.373 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:52 smithi089 bash[21793]: cluster 2024-07-15T06:42:50.912746+0000 mgr.y (mgr.14150) 186 : cluster [DBG] pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-07-15T06:42:54.308 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:54 smithi134 bash[24068]: cluster 2024-07-15T06:42:52.913394+0000 mgr.y (mgr.14150) 187 : cluster [DBG] pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-07-15T06:42:54.372 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:54 smithi089 bash[31372]: cluster 2024-07-15T06:42:52.913394+0000 mgr.y (mgr.14150) 187 : cluster [DBG] pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-07-15T06:42:54.373 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:54 smithi089 bash[21793]: cluster 2024-07-15T06:42:52.913394+0000 mgr.y (mgr.14150) 187 : cluster [DBG] pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-07-15T06:42:56.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:56 smithi134 bash[24068]: cluster 2024-07-15T06:42:54.914035+0000 mgr.y (mgr.14150) 188 : cluster [DBG] pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-07-15T06:42:56.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:56 smithi089 bash[21793]: cluster 2024-07-15T06:42:54.914035+0000 mgr.y (mgr.14150) 188 : cluster [DBG] pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-07-15T06:42:56.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:56 smithi089 bash[31372]: cluster 2024-07-15T06:42:54.914035+0000 mgr.y (mgr.14150) 188 : cluster [DBG] pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-07-15T06:42:58.266 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:42:58 smithi089 bash[21793]: cluster 2024-07-15T06:42:56.914587+0000 mgr.y (mgr.14150) 189 : cluster [DBG] pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-07-15T06:42:58.266 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:42:58 smithi089 bash[31372]: cluster 2024-07-15T06:42:56.914587+0000 mgr.y (mgr.14150) 189 : cluster [DBG] pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-07-15T06:42:58.308 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:42:58 smithi134 bash[24068]: cluster 2024-07-15T06:42:56.914587+0000 mgr.y (mgr.14150) 189 : cluster [DBG] pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-07-15T06:43:00.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:43:00 smithi134 bash[24068]: cluster 2024-07-15T06:42:58.915093+0000 mgr.y (mgr.14150) 190 : cluster [DBG] pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-07-15T06:43:00.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:43:00 smithi134 bash[24068]: audit 2024-07-15T06:42:59.261043+0000 mon.a (mon.0) 401 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-07-15T06:43:00.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:43:00 smithi134 bash[24068]: audit 2024-07-15T06:42:59.262864+0000 mon.a (mon.0) 402 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:43:00.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:43:00 smithi134 bash[24068]: cephadm 2024-07-15T06:42:59.264334+0000 mgr.y (mgr.14150) 191 : cephadm [INF] Deploying daemon osd.3 on smithi089 2024-07-15T06:43:00.373 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:43:00 smithi089 bash[21793]: cluster 2024-07-15T06:42:58.915093+0000 mgr.y (mgr.14150) 190 : cluster [DBG] pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-07-15T06:43:00.373 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:43:00 smithi089 bash[21793]: audit 2024-07-15T06:42:59.261043+0000 mon.a (mon.0) 401 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-07-15T06:43:00.373 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:43:00 smithi089 bash[21793]: audit 2024-07-15T06:42:59.262864+0000 mon.a (mon.0) 402 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:43:00.373 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:43:00 smithi089 bash[21793]: cephadm 2024-07-15T06:42:59.264334+0000 mgr.y (mgr.14150) 191 : cephadm [INF] Deploying daemon osd.3 on smithi089 2024-07-15T06:43:00.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:43:00 smithi089 bash[31372]: cluster 2024-07-15T06:42:58.915093+0000 mgr.y (mgr.14150) 190 : cluster [DBG] pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-07-15T06:43:00.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:43:00 smithi089 bash[31372]: audit 2024-07-15T06:42:59.261043+0000 mon.a (mon.0) 401 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-07-15T06:43:00.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:43:00 smithi089 bash[31372]: audit 2024-07-15T06:42:59.262864+0000 mon.a (mon.0) 402 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:43:00.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:43:00 smithi089 bash[31372]: cephadm 2024-07-15T06:42:59.264334+0000 mgr.y (mgr.14150) 191 : cephadm [INF] Deploying daemon osd.3 on smithi089 2024-07-15T06:43:02.308 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:43:02 smithi134 bash[24068]: cluster 2024-07-15T06:43:00.915668+0000 mgr.y (mgr.14150) 192 : cluster [DBG] pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-07-15T06:43:02.344 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:43:02 smithi089 bash[21793]: cluster 2024-07-15T06:43:00.915668+0000 mgr.y (mgr.14150) 192 : cluster [DBG] pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-07-15T06:43:02.345 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:43:02 smithi089 bash[31372]: cluster 2024-07-15T06:43:00.915668+0000 mgr.y (mgr.14150) 192 : cluster [DBG] pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-07-15T06:43:03.524 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:43:03 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:43:03.525 INFO:journalctl@ceph.osd.1.smithi089.stdout:Jul 15 06:43:03 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:43:03.525 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:43:03 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:43:03.525 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:43:03 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:43:03.525 INFO:journalctl@ceph.osd.2.smithi089.stdout:Jul 15 06:43:03 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:43:03.526 INFO:journalctl@ceph.osd.0.smithi089.stdout:Jul 15 06:43:03 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:43:03.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:43:03 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:43:03.872 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:43:03 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:43:03.873 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:43:03 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:43:03.873 INFO:journalctl@ceph.osd.0.smithi089.stdout:Jul 15 06:43:03 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:43:03.873 INFO:journalctl@ceph.osd.1.smithi089.stdout:Jul 15 06:43:03 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:43:03.874 INFO:journalctl@ceph.osd.2.smithi089.stdout:Jul 15 06:43:03 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:43:04.308 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:43:04 smithi134 bash[24068]: cluster 2024-07-15T06:43:02.916185+0000 mgr.y (mgr.14150) 193 : cluster [DBG] pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-07-15T06:43:04.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:43:04 smithi134 bash[24068]: audit 2024-07-15T06:43:03.804995+0000 mon.a (mon.0) 403 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:43:04.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:43:04 smithi134 bash[24068]: audit 2024-07-15T06:43:03.808822+0000 mon.a (mon.0) 404 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:43:04.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:43:04 smithi134 bash[24068]: audit 2024-07-15T06:43:03.815495+0000 mon.a (mon.0) 405 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:43:04.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:43:04 smithi089 bash[21793]: cluster 2024-07-15T06:43:02.916185+0000 mgr.y (mgr.14150) 193 : cluster [DBG] pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-07-15T06:43:04.373 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:43:04 smithi089 bash[21793]: audit 2024-07-15T06:43:03.804995+0000 mon.a (mon.0) 403 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:43:04.373 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:43:04 smithi089 bash[21793]: audit 2024-07-15T06:43:03.808822+0000 mon.a (mon.0) 404 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:43:04.373 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:43:04 smithi089 bash[21793]: audit 2024-07-15T06:43:03.815495+0000 mon.a (mon.0) 405 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:43:04.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:43:04 smithi089 bash[31372]: cluster 2024-07-15T06:43:02.916185+0000 mgr.y (mgr.14150) 193 : cluster [DBG] pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-07-15T06:43:04.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:43:04 smithi089 bash[31372]: audit 2024-07-15T06:43:03.804995+0000 mon.a (mon.0) 403 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:43:04.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:43:04 smithi089 bash[31372]: audit 2024-07-15T06:43:03.808822+0000 mon.a (mon.0) 404 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:43:04.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:43:04 smithi089 bash[31372]: audit 2024-07-15T06:43:03.815495+0000 mon.a (mon.0) 405 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:43:06.308 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:43:06 smithi134 bash[24068]: cluster 2024-07-15T06:43:04.916685+0000 mgr.y (mgr.14150) 194 : cluster [DBG] pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-07-15T06:43:06.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:43:06 smithi089 bash[21793]: cluster 2024-07-15T06:43:04.916685+0000 mgr.y (mgr.14150) 194 : cluster [DBG] pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-07-15T06:43:06.372 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:43:06 smithi089 bash[31372]: cluster 2024-07-15T06:43:04.916685+0000 mgr.y (mgr.14150) 194 : cluster [DBG] pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-07-15T06:43:08.308 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:43:08 smithi134 bash[24068]: cluster 2024-07-15T06:43:06.917250+0000 mgr.y (mgr.14150) 195 : cluster [DBG] pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-07-15T06:43:08.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:43:08 smithi089 bash[21793]: cluster 2024-07-15T06:43:06.917250+0000 mgr.y (mgr.14150) 195 : cluster [DBG] pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-07-15T06:43:08.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:43:08 smithi089 bash[31372]: cluster 2024-07-15T06:43:06.917250+0000 mgr.y (mgr.14150) 195 : cluster [DBG] pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-07-15T06:43:10.308 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:43:10 smithi134 bash[24068]: cluster 2024-07-15T06:43:08.917801+0000 mgr.y (mgr.14150) 196 : cluster [DBG] pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-07-15T06:43:10.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:43:10 smithi134 bash[24068]: audit 2024-07-15T06:43:09.892248+0000 mon.a (mon.0) 406 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:43:10.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:43:10 smithi134 bash[24068]: audit 2024-07-15T06:43:09.896436+0000 mon.a (mon.0) 407 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:43:10.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:43:10 smithi134 bash[24068]: audit 2024-07-15T06:43:09.897211+0000 mon.a (mon.0) 408 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:43:10.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:43:10 smithi134 bash[24068]: audit 2024-07-15T06:43:09.897886+0000 mon.a (mon.0) 409 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:43:10.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:43:10 smithi134 bash[24068]: audit 2024-07-15T06:43:09.902307+0000 mon.a (mon.0) 410 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:43:10.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:43:10 smithi089 bash[21793]: cluster 2024-07-15T06:43:08.917801+0000 mgr.y (mgr.14150) 196 : cluster [DBG] pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-07-15T06:43:10.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:43:10 smithi089 bash[21793]: audit 2024-07-15T06:43:09.892248+0000 mon.a (mon.0) 406 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:43:10.373 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:43:10 smithi089 bash[21793]: audit 2024-07-15T06:43:09.896436+0000 mon.a (mon.0) 407 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:43:10.373 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:43:10 smithi089 bash[21793]: audit 2024-07-15T06:43:09.897211+0000 mon.a (mon.0) 408 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:43:10.373 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:43:10 smithi089 bash[21793]: audit 2024-07-15T06:43:09.897886+0000 mon.a (mon.0) 409 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:43:10.373 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:43:10 smithi089 bash[21793]: audit 2024-07-15T06:43:09.902307+0000 mon.a (mon.0) 410 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:43:10.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:43:10 smithi089 bash[31372]: cluster 2024-07-15T06:43:08.917801+0000 mgr.y (mgr.14150) 196 : cluster [DBG] pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-07-15T06:43:10.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:43:10 smithi089 bash[31372]: audit 2024-07-15T06:43:09.892248+0000 mon.a (mon.0) 406 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:43:10.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:43:10 smithi089 bash[31372]: audit 2024-07-15T06:43:09.896436+0000 mon.a (mon.0) 407 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:43:10.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:43:10 smithi089 bash[31372]: audit 2024-07-15T06:43:09.897211+0000 mon.a (mon.0) 408 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:43:10.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:43:10 smithi089 bash[31372]: audit 2024-07-15T06:43:09.897886+0000 mon.a (mon.0) 409 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:43:10.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:43:10 smithi089 bash[31372]: audit 2024-07-15T06:43:09.902307+0000 mon.a (mon.0) 410 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:43:11.217 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:43:11 smithi089 bash[21793]: audit 2024-07-15T06:43:10.650603+0000 mon.c (mon.2) 12 : audit [INF] from='osd.3 [v2:172.21.15.89:6826/4083663609,v1:172.21.15.89:6827/4083663609]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-07-15T06:43:11.217 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:43:11 smithi089 bash[21793]: audit 2024-07-15T06:43:10.650937+0000 mon.a (mon.0) 411 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-07-15T06:43:11.218 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:43:11 smithi089 bash[31372]: audit 2024-07-15T06:43:10.650603+0000 mon.c (mon.2) 12 : audit [INF] from='osd.3 [v2:172.21.15.89:6826/4083663609,v1:172.21.15.89:6827/4083663609]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-07-15T06:43:11.218 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:43:11 smithi089 bash[31372]: audit 2024-07-15T06:43:10.650937+0000 mon.a (mon.0) 411 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-07-15T06:43:11.308 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:43:11 smithi134 bash[24068]: audit 2024-07-15T06:43:10.650603+0000 mon.c (mon.2) 12 : audit [INF] from='osd.3 [v2:172.21.15.89:6826/4083663609,v1:172.21.15.89:6827/4083663609]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-07-15T06:43:11.308 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:43:11 smithi134 bash[24068]: audit 2024-07-15T06:43:10.650937+0000 mon.a (mon.0) 411 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-07-15T06:43:12.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:43:12 smithi089 bash[21793]: cluster 2024-07-15T06:43:10.918157+0000 mgr.y (mgr.14150) 197 : cluster [DBG] pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-07-15T06:43:12.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:43:12 smithi089 bash[21793]: audit 2024-07-15T06:43:11.050909+0000 mon.a (mon.0) 412 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-07-15T06:43:12.123 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:43:12 smithi089 bash[21793]: cluster 2024-07-15T06:43:11.050991+0000 mon.a (mon.0) 413 : cluster [DBG] osdmap e23: 4 total, 3 up, 4 in 2024-07-15T06:43:12.123 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:43:12 smithi089 bash[21793]: audit 2024-07-15T06:43:11.051139+0000 mon.a (mon.0) 414 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-07-15T06:43:12.123 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:43:12 smithi089 bash[21793]: audit 2024-07-15T06:43:11.051945+0000 mon.c (mon.2) 13 : audit [INF] from='osd.3 [v2:172.21.15.89:6826/4083663609,v1:172.21.15.89:6827/4083663609]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2024-07-15T06:43:12.123 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:43:12 smithi089 bash[21793]: audit 2024-07-15T06:43:11.052319+0000 mon.a (mon.0) 415 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2024-07-15T06:43:12.123 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:43:12 smithi089 bash[31372]: cluster 2024-07-15T06:43:10.918157+0000 mgr.y (mgr.14150) 197 : cluster [DBG] pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-07-15T06:43:12.123 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:43:12 smithi089 bash[31372]: audit 2024-07-15T06:43:11.050909+0000 mon.a (mon.0) 412 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-07-15T06:43:12.123 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:43:12 smithi089 bash[31372]: cluster 2024-07-15T06:43:11.050991+0000 mon.a (mon.0) 413 : cluster [DBG] osdmap e23: 4 total, 3 up, 4 in 2024-07-15T06:43:12.123 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:43:12 smithi089 bash[31372]: audit 2024-07-15T06:43:11.051139+0000 mon.a (mon.0) 414 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-07-15T06:43:12.123 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:43:12 smithi089 bash[31372]: audit 2024-07-15T06:43:11.051945+0000 mon.c (mon.2) 13 : audit [INF] from='osd.3 [v2:172.21.15.89:6826/4083663609,v1:172.21.15.89:6827/4083663609]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2024-07-15T06:43:12.123 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:43:12 smithi089 bash[31372]: audit 2024-07-15T06:43:11.052319+0000 mon.a (mon.0) 415 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2024-07-15T06:43:12.308 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:43:12 smithi134 bash[24068]: cluster 2024-07-15T06:43:10.918157+0000 mgr.y (mgr.14150) 197 : cluster [DBG] pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-07-15T06:43:12.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:43:12 smithi134 bash[24068]: audit 2024-07-15T06:43:11.050909+0000 mon.a (mon.0) 412 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-07-15T06:43:12.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:43:12 smithi134 bash[24068]: cluster 2024-07-15T06:43:11.050991+0000 mon.a (mon.0) 413 : cluster [DBG] osdmap e23: 4 total, 3 up, 4 in 2024-07-15T06:43:12.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:43:12 smithi134 bash[24068]: audit 2024-07-15T06:43:11.051139+0000 mon.a (mon.0) 414 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-07-15T06:43:12.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:43:12 smithi134 bash[24068]: audit 2024-07-15T06:43:11.051945+0000 mon.c (mon.2) 13 : audit [INF] from='osd.3 [v2:172.21.15.89:6826/4083663609,v1:172.21.15.89:6827/4083663609]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2024-07-15T06:43:12.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:43:12 smithi134 bash[24068]: audit 2024-07-15T06:43:11.052319+0000 mon.a (mon.0) 415 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2024-07-15T06:43:13.328 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:43:13 smithi089 bash[21793]: audit 2024-07-15T06:43:12.056397+0000 mon.a (mon.0) 416 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]': finished 2024-07-15T06:43:13.328 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:43:13 smithi089 bash[21793]: cluster 2024-07-15T06:43:12.056552+0000 mon.a (mon.0) 417 : cluster [DBG] osdmap e24: 4 total, 3 up, 4 in 2024-07-15T06:43:13.328 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:43:13 smithi089 bash[21793]: audit 2024-07-15T06:43:12.057220+0000 mon.a (mon.0) 418 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-07-15T06:43:13.328 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:43:13 smithi089 bash[21793]: audit 2024-07-15T06:43:12.066043+0000 mon.a (mon.0) 419 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-07-15T06:43:13.328 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:43:13 smithi089 bash[31372]: audit 2024-07-15T06:43:12.056397+0000 mon.a (mon.0) 416 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]': finished 2024-07-15T06:43:13.328 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:43:13 smithi089 bash[31372]: cluster 2024-07-15T06:43:12.056552+0000 mon.a (mon.0) 417 : cluster [DBG] osdmap e24: 4 total, 3 up, 4 in 2024-07-15T06:43:13.328 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:43:13 smithi089 bash[31372]: audit 2024-07-15T06:43:12.057220+0000 mon.a (mon.0) 418 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-07-15T06:43:13.329 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:43:13 smithi089 bash[31372]: audit 2024-07-15T06:43:12.066043+0000 mon.a (mon.0) 419 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-07-15T06:43:13.558 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:43:13 smithi134 bash[24068]: audit 2024-07-15T06:43:12.056397+0000 mon.a (mon.0) 416 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]': finished 2024-07-15T06:43:13.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:43:13 smithi134 bash[24068]: cluster 2024-07-15T06:43:12.056552+0000 mon.a (mon.0) 417 : cluster [DBG] osdmap e24: 4 total, 3 up, 4 in 2024-07-15T06:43:13.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:43:13 smithi134 bash[24068]: audit 2024-07-15T06:43:12.057220+0000 mon.a (mon.0) 418 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-07-15T06:43:13.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:43:13 smithi134 bash[24068]: audit 2024-07-15T06:43:12.066043+0000 mon.a (mon.0) 419 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-07-15T06:43:14.064 INFO:teuthology.orchestra.run.smithi089.stdout:Created osd(s) 3 on host 'smithi089' 2024-07-15T06:43:14.358 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:43:14 smithi089 bash[31372]: cluster 2024-07-15T06:43:11.604013+0000 osd.3 (osd.3) 1 : cluster [DBG] purged_snaps scrub starts 2024-07-15T06:43:14.358 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:43:14 smithi089 bash[31372]: cluster 2024-07-15T06:43:11.604107+0000 osd.3 (osd.3) 2 : cluster [DBG] purged_snaps scrub ok 2024-07-15T06:43:14.358 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:43:14 smithi089 bash[31372]: cluster 2024-07-15T06:43:12.918625+0000 mgr.y (mgr.14150) 198 : cluster [DBG] pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-07-15T06:43:14.358 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:43:14 smithi089 bash[31372]: cluster 2024-07-15T06:43:13.064470+0000 mon.a (mon.0) 420 : cluster [INF] osd.3 [v2:172.21.15.89:6826/4083663609,v1:172.21.15.89:6827/4083663609] boot 2024-07-15T06:43:14.358 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:43:14 smithi089 bash[31372]: cluster 2024-07-15T06:43:13.064514+0000 mon.a (mon.0) 421 : cluster [DBG] osdmap e25: 4 total, 4 up, 4 in 2024-07-15T06:43:14.358 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:43:14 smithi089 bash[31372]: audit 2024-07-15T06:43:13.064930+0000 mon.a (mon.0) 422 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-07-15T06:43:14.359 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:43:14 smithi089 bash[31372]: audit 2024-07-15T06:43:14.046284+0000 mon.a (mon.0) 423 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:43:14.359 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:43:14 smithi089 bash[31372]: audit 2024-07-15T06:43:14.052014+0000 mon.a (mon.0) 424 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:43:14.359 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:43:14 smithi089 bash[31372]: audit 2024-07-15T06:43:14.062451+0000 mon.a (mon.0) 425 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:43:14.359 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:43:14 smithi089 bash[21793]: cluster 2024-07-15T06:43:11.604013+0000 osd.3 (osd.3) 1 : cluster [DBG] purged_snaps scrub starts 2024-07-15T06:43:14.359 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:43:14 smithi089 bash[21793]: cluster 2024-07-15T06:43:11.604107+0000 osd.3 (osd.3) 2 : cluster [DBG] purged_snaps scrub ok 2024-07-15T06:43:14.359 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:43:14 smithi089 bash[21793]: cluster 2024-07-15T06:43:12.918625+0000 mgr.y (mgr.14150) 198 : cluster [DBG] pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-07-15T06:43:14.359 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:43:14 smithi089 bash[21793]: cluster 2024-07-15T06:43:13.064470+0000 mon.a (mon.0) 420 : cluster [INF] osd.3 [v2:172.21.15.89:6826/4083663609,v1:172.21.15.89:6827/4083663609] boot 2024-07-15T06:43:14.359 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:43:14 smithi089 bash[21793]: cluster 2024-07-15T06:43:13.064514+0000 mon.a (mon.0) 421 : cluster [DBG] osdmap e25: 4 total, 4 up, 4 in 2024-07-15T06:43:14.359 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:43:14 smithi089 bash[21793]: audit 2024-07-15T06:43:13.064930+0000 mon.a (mon.0) 422 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-07-15T06:43:14.359 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:43:14 smithi089 bash[21793]: audit 2024-07-15T06:43:14.046284+0000 mon.a (mon.0) 423 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:43:14.359 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:43:14 smithi089 bash[21793]: audit 2024-07-15T06:43:14.052014+0000 mon.a (mon.0) 424 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:43:14.359 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:43:14 smithi089 bash[21793]: audit 2024-07-15T06:43:14.062451+0000 mon.a (mon.0) 425 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:43:14.558 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:43:14 smithi134 bash[24068]: cluster 2024-07-15T06:43:11.604013+0000 osd.3 (osd.3) 1 : cluster [DBG] purged_snaps scrub starts 2024-07-15T06:43:14.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:43:14 smithi134 bash[24068]: cluster 2024-07-15T06:43:11.604107+0000 osd.3 (osd.3) 2 : cluster [DBG] purged_snaps scrub ok 2024-07-15T06:43:14.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:43:14 smithi134 bash[24068]: cluster 2024-07-15T06:43:12.918625+0000 mgr.y (mgr.14150) 198 : cluster [DBG] pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-07-15T06:43:14.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:43:14 smithi134 bash[24068]: cluster 2024-07-15T06:43:13.064470+0000 mon.a (mon.0) 420 : cluster [INF] osd.3 [v2:172.21.15.89:6826/4083663609,v1:172.21.15.89:6827/4083663609] boot 2024-07-15T06:43:14.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:43:14 smithi134 bash[24068]: cluster 2024-07-15T06:43:13.064514+0000 mon.a (mon.0) 421 : cluster [DBG] osdmap e25: 4 total, 4 up, 4 in 2024-07-15T06:43:14.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:43:14 smithi134 bash[24068]: audit 2024-07-15T06:43:13.064930+0000 mon.a (mon.0) 422 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-07-15T06:43:14.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:43:14 smithi134 bash[24068]: audit 2024-07-15T06:43:14.046284+0000 mon.a (mon.0) 423 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:43:14.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:43:14 smithi134 bash[24068]: audit 2024-07-15T06:43:14.052014+0000 mon.a (mon.0) 424 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:43:14.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:43:14 smithi134 bash[24068]: audit 2024-07-15T06:43:14.062451+0000 mon.a (mon.0) 425 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:43:14.802 DEBUG:teuthology.orchestra.run.smithi089:osd.3> sudo journalctl -f -n 0 -u ceph-75ba02bc-4274-11ef-bcad-c7b262605968@osd.3.service 2024-07-15T06:43:14.846 INFO:tasks.cephadm:Deploying osd.4 on smithi134 with /dev/vg_nvme/lv_4... 2024-07-15T06:43:14.846 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-07-15T06:43:15.308 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:43:15 smithi134 bash[24068]: cluster 2024-07-15T06:43:14.084865+0000 mon.a (mon.0) 426 : cluster [DBG] osdmap e26: 4 total, 4 up, 4 in 2024-07-15T06:43:15.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:43:15 smithi089 bash[21793]: cluster 2024-07-15T06:43:14.084865+0000 mon.a (mon.0) 426 : cluster [DBG] osdmap e26: 4 total, 4 up, 4 in 2024-07-15T06:43:15.372 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:43:15 smithi089 bash[31372]: cluster 2024-07-15T06:43:14.084865+0000 mon.a (mon.0) 426 : cluster [DBG] osdmap e26: 4 total, 4 up, 4 in 2024-07-15T06:43:16.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:43:16 smithi089 bash[21793]: cluster 2024-07-15T06:43:14.919095+0000 mgr.y (mgr.14150) 199 : cluster [DBG] pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-15T06:43:16.372 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:43:16 smithi089 bash[31372]: cluster 2024-07-15T06:43:14.919095+0000 mgr.y (mgr.14150) 199 : cluster [DBG] pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-15T06:43:16.558 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:43:16 smithi134 bash[24068]: cluster 2024-07-15T06:43:14.919095+0000 mgr.y (mgr.14150) 199 : cluster [DBG] pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-15T06:43:18.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:43:18 smithi089 bash[21793]: cluster 2024-07-15T06:43:16.919624+0000 mgr.y (mgr.14150) 200 : cluster [DBG] pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-15T06:43:18.372 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:43:18 smithi089 bash[31372]: cluster 2024-07-15T06:43:16.919624+0000 mgr.y (mgr.14150) 200 : cluster [DBG] pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-15T06:43:18.558 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:43:18 smithi134 bash[24068]: cluster 2024-07-15T06:43:16.919624+0000 mgr.y (mgr.14150) 200 : cluster [DBG] pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-15T06:43:19.560 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.b/config 2024-07-15T06:43:20.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:43:20 smithi089 bash[21793]: cluster 2024-07-15T06:43:18.920171+0000 mgr.y (mgr.14150) 201 : cluster [DBG] pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-15T06:43:20.372 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:43:20 smithi089 bash[31372]: cluster 2024-07-15T06:43:18.920171+0000 mgr.y (mgr.14150) 201 : cluster [DBG] pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-15T06:43:20.558 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:43:20 smithi134 bash[24068]: cluster 2024-07-15T06:43:18.920171+0000 mgr.y (mgr.14150) 201 : cluster [DBG] pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-15T06:43:22.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:43:22 smithi089 bash[21793]: cluster 2024-07-15T06:43:20.920865+0000 mgr.y (mgr.14150) 202 : cluster [DBG] pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-15T06:43:22.372 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:43:22 smithi089 bash[31372]: cluster 2024-07-15T06:43:20.920865+0000 mgr.y (mgr.14150) 202 : cluster [DBG] pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-15T06:43:22.504 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:43:22 smithi134 bash[24068]: cluster 2024-07-15T06:43:20.920865+0000 mgr.y (mgr.14150) 202 : cluster [DBG] pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-15T06:43:22.834 INFO:teuthology.orchestra.run.smithi134.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-07-15T06:43:23.015 INFO:teuthology.orchestra.run.smithi134.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-07-15T06:43:23.032 INFO:teuthology.orchestra.run.smithi134.stderr: stderr: 10+0 records in 2024-07-15T06:43:23.032 INFO:teuthology.orchestra.run.smithi134.stderr:10+0 records out 2024-07-15T06:43:23.033 INFO:teuthology.orchestra.run.smithi134.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0149977 s, 699 MB/s 2024-07-15T06:43:23.033 INFO:teuthology.orchestra.run.smithi134.stderr:--> Zapping successful for: 2024-07-15T06:43:23.650 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph orch daemon add osd smithi134:vg_nvme/lv_4 2024-07-15T06:43:24.373 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:43:24 smithi089 bash[21793]: cluster 2024-07-15T06:43:22.921246+0000 mgr.y (mgr.14150) 203 : cluster [DBG] pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-15T06:43:24.373 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:43:24 smithi089 bash[21793]: audit 2024-07-15T06:43:23.773122+0000 mon.a (mon.0) 427 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:43:24.373 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:43:24 smithi089 bash[21793]: audit 2024-07-15T06:43:23.779837+0000 mon.a (mon.0) 428 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:43:24.373 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:43:24 smithi089 bash[21793]: audit 2024-07-15T06:43:23.781763+0000 mon.a (mon.0) 429 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:43:24.373 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:43:24 smithi089 bash[21793]: audit 2024-07-15T06:43:23.783072+0000 mon.a (mon.0) 430 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:43:24.373 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:43:24 smithi089 bash[21793]: audit 2024-07-15T06:43:23.784510+0000 mon.a (mon.0) 431 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:43:24.373 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:43:24 smithi089 bash[21793]: audit 2024-07-15T06:43:23.785666+0000 mon.a (mon.0) 432 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:43:24.373 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:43:24 smithi089 bash[21793]: audit 2024-07-15T06:43:23.790679+0000 mon.a (mon.0) 433 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:43:24.373 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:43:24 smithi089 bash[21793]: audit 2024-07-15T06:43:23.792954+0000 mon.a (mon.0) 434 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:43:24.373 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:43:24 smithi089 bash[21793]: audit 2024-07-15T06:43:23.794648+0000 mon.a (mon.0) 435 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:43:24.374 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:43:24 smithi089 bash[21793]: audit 2024-07-15T06:43:23.805086+0000 mon.a (mon.0) 436 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:43:24.374 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:43:24 smithi089 bash[31372]: cluster 2024-07-15T06:43:22.921246+0000 mgr.y (mgr.14150) 203 : cluster [DBG] pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-15T06:43:24.374 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:43:24 smithi089 bash[31372]: audit 2024-07-15T06:43:23.773122+0000 mon.a (mon.0) 427 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:43:24.374 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:43:24 smithi089 bash[31372]: audit 2024-07-15T06:43:23.779837+0000 mon.a (mon.0) 428 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:43:24.374 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:43:24 smithi089 bash[31372]: audit 2024-07-15T06:43:23.781763+0000 mon.a (mon.0) 429 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:43:24.374 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:43:24 smithi089 bash[31372]: audit 2024-07-15T06:43:23.783072+0000 mon.a (mon.0) 430 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:43:24.374 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:43:24 smithi089 bash[31372]: audit 2024-07-15T06:43:23.784510+0000 mon.a (mon.0) 431 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:43:24.374 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:43:24 smithi089 bash[31372]: audit 2024-07-15T06:43:23.785666+0000 mon.a (mon.0) 432 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:43:24.374 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:43:24 smithi089 bash[31372]: audit 2024-07-15T06:43:23.790679+0000 mon.a (mon.0) 433 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:43:24.374 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:43:24 smithi089 bash[31372]: audit 2024-07-15T06:43:23.792954+0000 mon.a (mon.0) 434 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:43:24.374 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:43:24 smithi089 bash[31372]: audit 2024-07-15T06:43:23.794648+0000 mon.a (mon.0) 435 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:43:24.375 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:43:24 smithi089 bash[31372]: audit 2024-07-15T06:43:23.805086+0000 mon.a (mon.0) 436 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:43:24.558 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:43:24 smithi134 bash[24068]: cluster 2024-07-15T06:43:22.921246+0000 mgr.y (mgr.14150) 203 : cluster [DBG] pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-15T06:43:24.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:43:24 smithi134 bash[24068]: audit 2024-07-15T06:43:23.773122+0000 mon.a (mon.0) 427 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:43:24.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:43:24 smithi134 bash[24068]: audit 2024-07-15T06:43:23.779837+0000 mon.a (mon.0) 428 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:43:24.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:43:24 smithi134 bash[24068]: audit 2024-07-15T06:43:23.781763+0000 mon.a (mon.0) 429 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:43:24.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:43:24 smithi134 bash[24068]: audit 2024-07-15T06:43:23.783072+0000 mon.a (mon.0) 430 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:43:24.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:43:24 smithi134 bash[24068]: audit 2024-07-15T06:43:23.784510+0000 mon.a (mon.0) 431 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:43:24.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:43:24 smithi134 bash[24068]: audit 2024-07-15T06:43:23.785666+0000 mon.a (mon.0) 432 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:43:24.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:43:24 smithi134 bash[24068]: audit 2024-07-15T06:43:23.790679+0000 mon.a (mon.0) 433 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:43:24.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:43:24 smithi134 bash[24068]: audit 2024-07-15T06:43:23.792954+0000 mon.a (mon.0) 434 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:43:24.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:43:24 smithi134 bash[24068]: audit 2024-07-15T06:43:23.794648+0000 mon.a (mon.0) 435 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:43:24.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:43:24 smithi134 bash[24068]: audit 2024-07-15T06:43:23.805086+0000 mon.a (mon.0) 436 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:43:25.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:43:25 smithi089 bash[21793]: cephadm 2024-07-15T06:43:23.766568+0000 mgr.y (mgr.14150) 204 : cephadm [INF] Detected new or changed devices on smithi089 2024-07-15T06:43:25.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:43:25 smithi089 bash[21793]: cephadm 2024-07-15T06:43:23.786555+0000 mgr.y (mgr.14150) 205 : cephadm [INF] Adjusting osd_memory_target on smithi089 to 4060M 2024-07-15T06:43:25.372 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:43:25 smithi089 bash[31372]: cephadm 2024-07-15T06:43:23.766568+0000 mgr.y (mgr.14150) 204 : cephadm [INF] Detected new or changed devices on smithi089 2024-07-15T06:43:25.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:43:25 smithi089 bash[31372]: cephadm 2024-07-15T06:43:23.786555+0000 mgr.y (mgr.14150) 205 : cephadm [INF] Adjusting osd_memory_target on smithi089 to 4060M 2024-07-15T06:43:25.558 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:43:25 smithi134 bash[24068]: cephadm 2024-07-15T06:43:23.766568+0000 mgr.y (mgr.14150) 204 : cephadm [INF] Detected new or changed devices on smithi089 2024-07-15T06:43:25.558 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:43:25 smithi134 bash[24068]: cephadm 2024-07-15T06:43:23.786555+0000 mgr.y (mgr.14150) 205 : cephadm [INF] Adjusting osd_memory_target on smithi089 to 4060M 2024-07-15T06:43:26.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:43:26 smithi089 bash[21793]: cluster 2024-07-15T06:43:24.921765+0000 mgr.y (mgr.14150) 206 : cluster [DBG] pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-15T06:43:26.372 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:43:26 smithi089 bash[31372]: cluster 2024-07-15T06:43:24.921765+0000 mgr.y (mgr.14150) 206 : cluster [DBG] pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-15T06:43:26.558 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:43:26 smithi134 bash[24068]: cluster 2024-07-15T06:43:24.921765+0000 mgr.y (mgr.14150) 206 : cluster [DBG] pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-15T06:43:28.366 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.b/config 2024-07-15T06:43:28.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:43:28 smithi089 bash[21793]: cluster 2024-07-15T06:43:26.922297+0000 mgr.y (mgr.14150) 207 : cluster [DBG] pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-15T06:43:28.372 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:43:28 smithi089 bash[31372]: cluster 2024-07-15T06:43:26.922297+0000 mgr.y (mgr.14150) 207 : cluster [DBG] pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-15T06:43:28.445 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:43:28 smithi134 bash[24068]: cluster 2024-07-15T06:43:26.922297+0000 mgr.y (mgr.14150) 207 : cluster [DBG] pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-15T06:43:30.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:43:30 smithi089 bash[21793]: cluster 2024-07-15T06:43:28.922813+0000 mgr.y (mgr.14150) 208 : cluster [DBG] pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-15T06:43:30.372 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:43:30 smithi089 bash[31372]: cluster 2024-07-15T06:43:28.922813+0000 mgr.y (mgr.14150) 208 : cluster [DBG] pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-15T06:43:30.558 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:43:30 smithi134 bash[24068]: cluster 2024-07-15T06:43:28.922813+0000 mgr.y (mgr.14150) 208 : cluster [DBG] pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-15T06:43:32.558 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:43:32 smithi134 bash[24068]: cluster 2024-07-15T06:43:30.923381+0000 mgr.y (mgr.14150) 209 : cluster [DBG] pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-15T06:43:32.558 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:43:32 smithi134 bash[24068]: audit 2024-07-15T06:43:31.233711+0000 mgr.y (mgr.14150) 210 : audit [DBG] from='client.24178 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi134:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T06:43:32.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:43:32 smithi134 bash[24068]: audit 2024-07-15T06:43:31.240567+0000 mon.a (mon.0) 437 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-07-15T06:43:32.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:43:32 smithi134 bash[24068]: audit 2024-07-15T06:43:31.246577+0000 mon.a (mon.0) 438 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-07-15T06:43:32.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:43:32 smithi134 bash[24068]: audit 2024-07-15T06:43:31.248324+0000 mon.a (mon.0) 439 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:43:32.622 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:43:32 smithi089 bash[21793]: cluster 2024-07-15T06:43:30.923381+0000 mgr.y (mgr.14150) 209 : cluster [DBG] pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-15T06:43:32.622 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:43:32 smithi089 bash[21793]: audit 2024-07-15T06:43:31.233711+0000 mgr.y (mgr.14150) 210 : audit [DBG] from='client.24178 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi134:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T06:43:32.622 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:43:32 smithi089 bash[21793]: audit 2024-07-15T06:43:31.240567+0000 mon.a (mon.0) 437 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-07-15T06:43:32.622 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:43:32 smithi089 bash[21793]: audit 2024-07-15T06:43:31.246577+0000 mon.a (mon.0) 438 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-07-15T06:43:32.622 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:43:32 smithi089 bash[21793]: audit 2024-07-15T06:43:31.248324+0000 mon.a (mon.0) 439 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:43:32.623 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:43:32 smithi089 bash[31372]: cluster 2024-07-15T06:43:30.923381+0000 mgr.y (mgr.14150) 209 : cluster [DBG] pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-15T06:43:32.623 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:43:32 smithi089 bash[31372]: audit 2024-07-15T06:43:31.233711+0000 mgr.y (mgr.14150) 210 : audit [DBG] from='client.24178 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi134:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T06:43:32.623 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:43:32 smithi089 bash[31372]: audit 2024-07-15T06:43:31.240567+0000 mon.a (mon.0) 437 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-07-15T06:43:32.623 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:43:32 smithi089 bash[31372]: audit 2024-07-15T06:43:31.246577+0000 mon.a (mon.0) 438 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-07-15T06:43:32.623 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:43:32 smithi089 bash[31372]: audit 2024-07-15T06:43:31.248324+0000 mon.a (mon.0) 439 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:43:34.558 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:43:34 smithi134 bash[24068]: cluster 2024-07-15T06:43:32.923878+0000 mgr.y (mgr.14150) 211 : cluster [DBG] pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-15T06:43:34.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:43:34 smithi089 bash[31372]: cluster 2024-07-15T06:43:32.923878+0000 mgr.y (mgr.14150) 211 : cluster [DBG] pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-15T06:43:34.622 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:43:34 smithi089 bash[21793]: cluster 2024-07-15T06:43:32.923878+0000 mgr.y (mgr.14150) 211 : cluster [DBG] pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-15T06:43:36.558 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:43:36 smithi134 bash[24068]: cluster 2024-07-15T06:43:34.924426+0000 mgr.y (mgr.14150) 212 : cluster [DBG] pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-15T06:43:36.622 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:43:36 smithi089 bash[21793]: cluster 2024-07-15T06:43:34.924426+0000 mgr.y (mgr.14150) 212 : cluster [DBG] pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-15T06:43:36.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:43:36 smithi089 bash[31372]: cluster 2024-07-15T06:43:34.924426+0000 mgr.y (mgr.14150) 212 : cluster [DBG] pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-15T06:43:38.308 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:43:38 smithi134 bash[24068]: cluster 2024-07-15T06:43:36.924911+0000 mgr.y (mgr.14150) 213 : cluster [DBG] pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-15T06:43:38.622 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:43:38 smithi089 bash[21793]: cluster 2024-07-15T06:43:36.924911+0000 mgr.y (mgr.14150) 213 : cluster [DBG] pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-15T06:43:38.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:43:38 smithi089 bash[31372]: cluster 2024-07-15T06:43:36.924911+0000 mgr.y (mgr.14150) 213 : cluster [DBG] pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-15T06:43:40.558 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:43:40 smithi134 bash[24068]: cluster 2024-07-15T06:43:38.925433+0000 mgr.y (mgr.14150) 214 : cluster [DBG] pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-15T06:43:40.558 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:43:40 smithi134 bash[24068]: audit 2024-07-15T06:43:39.466994+0000 mon.b (mon.1) 9 : audit [INF] from='client.? 172.21.15.134:0/61228482' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b97874a6-e72a-4068-9c71-a50b8790be2c"}]: dispatch 2024-07-15T06:43:40.558 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:43:40 smithi134 bash[24068]: audit 2024-07-15T06:43:39.468890+0000 mon.a (mon.0) 440 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b97874a6-e72a-4068-9c71-a50b8790be2c"}]: dispatch 2024-07-15T06:43:40.558 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:43:40 smithi134 bash[24068]: audit 2024-07-15T06:43:39.477587+0000 mon.a (mon.0) 441 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b97874a6-e72a-4068-9c71-a50b8790be2c"}]': finished 2024-07-15T06:43:40.558 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:43:40 smithi134 bash[24068]: cluster 2024-07-15T06:43:39.477764+0000 mon.a (mon.0) 442 : cluster [DBG] osdmap e27: 5 total, 4 up, 5 in 2024-07-15T06:43:40.558 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:43:40 smithi134 bash[24068]: audit 2024-07-15T06:43:39.478057+0000 mon.a (mon.0) 443 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-07-15T06:43:40.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:43:40 smithi134 bash[24068]: audit 2024-07-15T06:43:40.088814+0000 mon.b (mon.1) 10 : audit [DBG] from='client.? 172.21.15.134:0/3049709207' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-07-15T06:43:40.622 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:43:40 smithi089 bash[21793]: cluster 2024-07-15T06:43:38.925433+0000 mgr.y (mgr.14150) 214 : cluster [DBG] pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-15T06:43:40.622 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:43:40 smithi089 bash[21793]: audit 2024-07-15T06:43:39.466994+0000 mon.b (mon.1) 9 : audit [INF] from='client.? 172.21.15.134:0/61228482' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b97874a6-e72a-4068-9c71-a50b8790be2c"}]: dispatch 2024-07-15T06:43:40.622 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:43:40 smithi089 bash[21793]: audit 2024-07-15T06:43:39.468890+0000 mon.a (mon.0) 440 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b97874a6-e72a-4068-9c71-a50b8790be2c"}]: dispatch 2024-07-15T06:43:40.622 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:43:40 smithi089 bash[21793]: audit 2024-07-15T06:43:39.477587+0000 mon.a (mon.0) 441 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b97874a6-e72a-4068-9c71-a50b8790be2c"}]': finished 2024-07-15T06:43:40.623 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:43:40 smithi089 bash[21793]: cluster 2024-07-15T06:43:39.477764+0000 mon.a (mon.0) 442 : cluster [DBG] osdmap e27: 5 total, 4 up, 5 in 2024-07-15T06:43:40.623 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:43:40 smithi089 bash[21793]: audit 2024-07-15T06:43:39.478057+0000 mon.a (mon.0) 443 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-07-15T06:43:40.623 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:43:40 smithi089 bash[21793]: audit 2024-07-15T06:43:40.088814+0000 mon.b (mon.1) 10 : audit [DBG] from='client.? 172.21.15.134:0/3049709207' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-07-15T06:43:40.623 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:43:40 smithi089 bash[31372]: cluster 2024-07-15T06:43:38.925433+0000 mgr.y (mgr.14150) 214 : cluster [DBG] pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-15T06:43:40.623 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:43:40 smithi089 bash[31372]: audit 2024-07-15T06:43:39.466994+0000 mon.b (mon.1) 9 : audit [INF] from='client.? 172.21.15.134:0/61228482' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b97874a6-e72a-4068-9c71-a50b8790be2c"}]: dispatch 2024-07-15T06:43:40.623 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:43:40 smithi089 bash[31372]: audit 2024-07-15T06:43:39.468890+0000 mon.a (mon.0) 440 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b97874a6-e72a-4068-9c71-a50b8790be2c"}]: dispatch 2024-07-15T06:43:40.623 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:43:40 smithi089 bash[31372]: audit 2024-07-15T06:43:39.477587+0000 mon.a (mon.0) 441 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b97874a6-e72a-4068-9c71-a50b8790be2c"}]': finished 2024-07-15T06:43:40.623 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:43:40 smithi089 bash[31372]: cluster 2024-07-15T06:43:39.477764+0000 mon.a (mon.0) 442 : cluster [DBG] osdmap e27: 5 total, 4 up, 5 in 2024-07-15T06:43:40.623 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:43:40 smithi089 bash[31372]: audit 2024-07-15T06:43:39.478057+0000 mon.a (mon.0) 443 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-07-15T06:43:40.623 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:43:40 smithi089 bash[31372]: audit 2024-07-15T06:43:40.088814+0000 mon.b (mon.1) 10 : audit [DBG] from='client.? 172.21.15.134:0/3049709207' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-07-15T06:43:42.480 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:43:42 smithi134 bash[24068]: cluster 2024-07-15T06:43:40.925995+0000 mgr.y (mgr.14150) 215 : cluster [DBG] pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-15T06:43:42.622 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:43:42 smithi089 bash[21793]: cluster 2024-07-15T06:43:40.925995+0000 mgr.y (mgr.14150) 215 : cluster [DBG] pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-15T06:43:42.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:43:42 smithi089 bash[31372]: cluster 2024-07-15T06:43:40.925995+0000 mgr.y (mgr.14150) 215 : cluster [DBG] pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-15T06:43:44.558 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:43:44 smithi134 bash[24068]: cluster 2024-07-15T06:43:42.926529+0000 mgr.y (mgr.14150) 216 : cluster [DBG] pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-15T06:43:44.622 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:43:44 smithi089 bash[21793]: cluster 2024-07-15T06:43:42.926529+0000 mgr.y (mgr.14150) 216 : cluster [DBG] pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-15T06:43:44.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:43:44 smithi089 bash[31372]: cluster 2024-07-15T06:43:42.926529+0000 mgr.y (mgr.14150) 216 : cluster [DBG] pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-15T06:43:46.558 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:43:46 smithi134 bash[24068]: cluster 2024-07-15T06:43:44.927147+0000 mgr.y (mgr.14150) 217 : cluster [DBG] pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-15T06:43:46.622 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:43:46 smithi089 bash[21793]: cluster 2024-07-15T06:43:44.927147+0000 mgr.y (mgr.14150) 217 : cluster [DBG] pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-15T06:43:46.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:43:46 smithi089 bash[31372]: cluster 2024-07-15T06:43:44.927147+0000 mgr.y (mgr.14150) 217 : cluster [DBG] pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-15T06:43:48.308 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:43:48 smithi134 bash[24068]: cluster 2024-07-15T06:43:46.927678+0000 mgr.y (mgr.14150) 218 : cluster [DBG] pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-15T06:43:48.622 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:43:48 smithi089 bash[21793]: cluster 2024-07-15T06:43:46.927678+0000 mgr.y (mgr.14150) 218 : cluster [DBG] pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-15T06:43:48.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:43:48 smithi089 bash[31372]: cluster 2024-07-15T06:43:46.927678+0000 mgr.y (mgr.14150) 218 : cluster [DBG] pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-15T06:43:50.557 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:43:50 smithi134 bash[24068]: cluster 2024-07-15T06:43:48.928193+0000 mgr.y (mgr.14150) 219 : cluster [DBG] pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-15T06:43:50.622 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:43:50 smithi089 bash[21793]: cluster 2024-07-15T06:43:48.928193+0000 mgr.y (mgr.14150) 219 : cluster [DBG] pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-15T06:43:50.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:43:50 smithi089 bash[31372]: cluster 2024-07-15T06:43:48.928193+0000 mgr.y (mgr.14150) 219 : cluster [DBG] pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-15T06:43:52.558 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:43:52 smithi134 bash[24068]: cluster 2024-07-15T06:43:50.928748+0000 mgr.y (mgr.14150) 220 : cluster [DBG] pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-15T06:43:52.558 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:43:52 smithi134 bash[24068]: audit 2024-07-15T06:43:51.570309+0000 mon.a (mon.0) 444 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-07-15T06:43:52.558 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:43:52 smithi134 bash[24068]: audit 2024-07-15T06:43:51.572094+0000 mon.a (mon.0) 445 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:43:52.622 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:43:52 smithi089 bash[21793]: cluster 2024-07-15T06:43:50.928748+0000 mgr.y (mgr.14150) 220 : cluster [DBG] pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-15T06:43:52.622 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:43:52 smithi089 bash[21793]: audit 2024-07-15T06:43:51.570309+0000 mon.a (mon.0) 444 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-07-15T06:43:52.622 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:43:52 smithi089 bash[21793]: audit 2024-07-15T06:43:51.572094+0000 mon.a (mon.0) 445 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:43:52.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:43:52 smithi089 bash[31372]: cluster 2024-07-15T06:43:50.928748+0000 mgr.y (mgr.14150) 220 : cluster [DBG] pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-15T06:43:52.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:43:52 smithi089 bash[31372]: audit 2024-07-15T06:43:51.570309+0000 mon.a (mon.0) 444 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-07-15T06:43:52.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:43:52 smithi089 bash[31372]: audit 2024-07-15T06:43:51.572094+0000 mon.a (mon.0) 445 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:43:53.558 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:43:53 smithi134 bash[24068]: cephadm 2024-07-15T06:43:51.573690+0000 mgr.y (mgr.14150) 221 : cephadm [INF] Deploying daemon osd.4 on smithi134 2024-07-15T06:43:53.622 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:43:53 smithi089 bash[21793]: cephadm 2024-07-15T06:43:51.573690+0000 mgr.y (mgr.14150) 221 : cephadm [INF] Deploying daemon osd.4 on smithi134 2024-07-15T06:43:53.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:43:53 smithi089 bash[31372]: cephadm 2024-07-15T06:43:51.573690+0000 mgr.y (mgr.14150) 221 : cephadm [INF] Deploying daemon osd.4 on smithi134 2024-07-15T06:43:54.558 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:43:54 smithi134 bash[24068]: cluster 2024-07-15T06:43:52.929279+0000 mgr.y (mgr.14150) 222 : cluster [DBG] pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-15T06:43:54.622 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:43:54 smithi089 bash[21793]: cluster 2024-07-15T06:43:52.929279+0000 mgr.y (mgr.14150) 222 : cluster [DBG] pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-15T06:43:54.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:43:54 smithi089 bash[31372]: cluster 2024-07-15T06:43:52.929279+0000 mgr.y (mgr.14150) 222 : cluster [DBG] pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-15T06:43:55.903 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:43:55 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:43:55.903 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:43:55 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:43:56.181 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:43:56 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:43:56.182 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:43:56 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:43:56.558 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:43:56 smithi134 bash[24068]: cluster 2024-07-15T06:43:54.929838+0000 mgr.y (mgr.14150) 223 : cluster [DBG] pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-15T06:43:56.558 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:43:56 smithi134 bash[24068]: audit 2024-07-15T06:43:56.166295+0000 mon.a (mon.0) 446 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:43:56.558 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:43:56 smithi134 bash[24068]: audit 2024-07-15T06:43:56.174002+0000 mon.a (mon.0) 447 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:43:56.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:43:56 smithi089 bash[31372]: cluster 2024-07-15T06:43:54.929838+0000 mgr.y (mgr.14150) 223 : cluster [DBG] pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-15T06:43:56.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:43:56 smithi089 bash[31372]: audit 2024-07-15T06:43:56.166295+0000 mon.a (mon.0) 446 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:43:56.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:43:56 smithi089 bash[31372]: audit 2024-07-15T06:43:56.174002+0000 mon.a (mon.0) 447 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:43:56.622 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:43:56 smithi089 bash[21793]: cluster 2024-07-15T06:43:54.929838+0000 mgr.y (mgr.14150) 223 : cluster [DBG] pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-15T06:43:56.622 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:43:56 smithi089 bash[21793]: audit 2024-07-15T06:43:56.166295+0000 mon.a (mon.0) 446 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:43:56.622 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:43:56 smithi089 bash[21793]: audit 2024-07-15T06:43:56.174002+0000 mon.a (mon.0) 447 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:43:57.537 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:43:57 smithi134 bash[24068]: audit 2024-07-15T06:43:56.182091+0000 mon.a (mon.0) 448 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:43:57.622 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:43:57 smithi089 bash[21793]: audit 2024-07-15T06:43:56.182091+0000 mon.a (mon.0) 448 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:43:57.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:43:57 smithi089 bash[31372]: audit 2024-07-15T06:43:56.182091+0000 mon.a (mon.0) 448 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:43:58.558 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:43:58 smithi134 bash[24068]: cluster 2024-07-15T06:43:56.930336+0000 mgr.y (mgr.14150) 224 : cluster [DBG] pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-15T06:43:58.622 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:43:58 smithi089 bash[21793]: cluster 2024-07-15T06:43:56.930336+0000 mgr.y (mgr.14150) 224 : cluster [DBG] pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-15T06:43:58.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:43:58 smithi089 bash[31372]: cluster 2024-07-15T06:43:56.930336+0000 mgr.y (mgr.14150) 224 : cluster [DBG] pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-15T06:44:00.308 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:00 smithi134 bash[24068]: cluster 2024-07-15T06:43:58.930877+0000 mgr.y (mgr.14150) 225 : cluster [DBG] pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-15T06:44:00.622 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:00 smithi089 bash[21793]: cluster 2024-07-15T06:43:58.930877+0000 mgr.y (mgr.14150) 225 : cluster [DBG] pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-15T06:44:00.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:00 smithi089 bash[31372]: cluster 2024-07-15T06:43:58.930877+0000 mgr.y (mgr.14150) 225 : cluster [DBG] pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-15T06:44:02.308 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:02 smithi134 bash[24068]: cluster 2024-07-15T06:44:00.931455+0000 mgr.y (mgr.14150) 226 : cluster [DBG] pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-15T06:44:02.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:02 smithi089 bash[21793]: cluster 2024-07-15T06:44:00.931455+0000 mgr.y (mgr.14150) 226 : cluster [DBG] pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-15T06:44:02.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:02 smithi089 bash[31372]: cluster 2024-07-15T06:44:00.931455+0000 mgr.y (mgr.14150) 226 : cluster [DBG] pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-15T06:44:03.479 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:03 smithi134 bash[24068]: audit 2024-07-15T06:44:02.213549+0000 mon.a (mon.0) 449 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:44:03.479 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:03 smithi134 bash[24068]: audit 2024-07-15T06:44:02.221477+0000 mon.a (mon.0) 450 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:44:03.479 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:03 smithi134 bash[24068]: audit 2024-07-15T06:44:02.223183+0000 mon.a (mon.0) 451 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:44:03.479 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:03 smithi134 bash[24068]: audit 2024-07-15T06:44:02.224909+0000 mon.a (mon.0) 452 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:44:03.479 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:03 smithi134 bash[24068]: audit 2024-07-15T06:44:02.234437+0000 mon.a (mon.0) 453 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:44:03.479 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:03 smithi134 bash[24068]: audit 2024-07-15T06:44:02.842683+0000 mon.b (mon.1) 11 : audit [INF] from='osd.4 [v2:172.21.15.134:6800/3588917663,v1:172.21.15.134:6801/3588917663]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-07-15T06:44:03.479 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:03 smithi134 bash[24068]: audit 2024-07-15T06:44:02.844297+0000 mon.a (mon.0) 454 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-07-15T06:44:03.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:03 smithi089 bash[31372]: audit 2024-07-15T06:44:02.213549+0000 mon.a (mon.0) 449 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:44:03.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:03 smithi089 bash[31372]: audit 2024-07-15T06:44:02.221477+0000 mon.a (mon.0) 450 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:44:03.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:03 smithi089 bash[31372]: audit 2024-07-15T06:44:02.223183+0000 mon.a (mon.0) 451 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:44:03.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:03 smithi089 bash[31372]: audit 2024-07-15T06:44:02.224909+0000 mon.a (mon.0) 452 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:44:03.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:03 smithi089 bash[31372]: audit 2024-07-15T06:44:02.234437+0000 mon.a (mon.0) 453 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:44:03.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:03 smithi089 bash[31372]: audit 2024-07-15T06:44:02.842683+0000 mon.b (mon.1) 11 : audit [INF] from='osd.4 [v2:172.21.15.134:6800/3588917663,v1:172.21.15.134:6801/3588917663]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-07-15T06:44:03.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:03 smithi089 bash[31372]: audit 2024-07-15T06:44:02.844297+0000 mon.a (mon.0) 454 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-07-15T06:44:03.623 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:03 smithi089 bash[21793]: audit 2024-07-15T06:44:02.213549+0000 mon.a (mon.0) 449 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:44:03.623 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:03 smithi089 bash[21793]: audit 2024-07-15T06:44:02.221477+0000 mon.a (mon.0) 450 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:44:03.623 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:03 smithi089 bash[21793]: audit 2024-07-15T06:44:02.223183+0000 mon.a (mon.0) 451 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:44:03.623 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:03 smithi089 bash[21793]: audit 2024-07-15T06:44:02.224909+0000 mon.a (mon.0) 452 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:44:03.623 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:03 smithi089 bash[21793]: audit 2024-07-15T06:44:02.234437+0000 mon.a (mon.0) 453 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:44:03.623 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:03 smithi089 bash[21793]: audit 2024-07-15T06:44:02.842683+0000 mon.b (mon.1) 11 : audit [INF] from='osd.4 [v2:172.21.15.134:6800/3588917663,v1:172.21.15.134:6801/3588917663]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-07-15T06:44:03.623 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:03 smithi089 bash[21793]: audit 2024-07-15T06:44:02.844297+0000 mon.a (mon.0) 454 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-07-15T06:44:04.558 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:04 smithi134 bash[24068]: cluster 2024-07-15T06:44:02.931929+0000 mgr.y (mgr.14150) 227 : cluster [DBG] pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-15T06:44:04.558 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:04 smithi134 bash[24068]: audit 2024-07-15T06:44:03.242778+0000 mon.a (mon.0) 455 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-07-15T06:44:04.558 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:04 smithi134 bash[24068]: audit 2024-07-15T06:44:03.242926+0000 mon.b (mon.1) 12 : audit [INF] from='osd.4 [v2:172.21.15.134:6800/3588917663,v1:172.21.15.134:6801/3588917663]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]: dispatch 2024-07-15T06:44:04.558 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:04 smithi134 bash[24068]: cluster 2024-07-15T06:44:03.242972+0000 mon.a (mon.0) 456 : cluster [DBG] osdmap e28: 5 total, 4 up, 5 in 2024-07-15T06:44:04.558 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:04 smithi134 bash[24068]: audit 2024-07-15T06:44:03.243225+0000 mon.a (mon.0) 457 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-07-15T06:44:04.558 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:04 smithi134 bash[24068]: audit 2024-07-15T06:44:03.244550+0000 mon.a (mon.0) 458 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]: dispatch 2024-07-15T06:44:04.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:04 smithi089 bash[31372]: cluster 2024-07-15T06:44:02.931929+0000 mgr.y (mgr.14150) 227 : cluster [DBG] pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-15T06:44:04.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:04 smithi089 bash[31372]: audit 2024-07-15T06:44:03.242778+0000 mon.a (mon.0) 455 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-07-15T06:44:04.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:04 smithi089 bash[31372]: audit 2024-07-15T06:44:03.242926+0000 mon.b (mon.1) 12 : audit [INF] from='osd.4 [v2:172.21.15.134:6800/3588917663,v1:172.21.15.134:6801/3588917663]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]: dispatch 2024-07-15T06:44:04.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:04 smithi089 bash[31372]: cluster 2024-07-15T06:44:03.242972+0000 mon.a (mon.0) 456 : cluster [DBG] osdmap e28: 5 total, 4 up, 5 in 2024-07-15T06:44:04.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:04 smithi089 bash[31372]: audit 2024-07-15T06:44:03.243225+0000 mon.a (mon.0) 457 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-07-15T06:44:04.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:04 smithi089 bash[31372]: audit 2024-07-15T06:44:03.244550+0000 mon.a (mon.0) 458 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]: dispatch 2024-07-15T06:44:04.622 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:04 smithi089 bash[21793]: cluster 2024-07-15T06:44:02.931929+0000 mgr.y (mgr.14150) 227 : cluster [DBG] pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-15T06:44:04.622 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:04 smithi089 bash[21793]: audit 2024-07-15T06:44:03.242778+0000 mon.a (mon.0) 455 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-07-15T06:44:04.623 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:04 smithi089 bash[21793]: audit 2024-07-15T06:44:03.242926+0000 mon.b (mon.1) 12 : audit [INF] from='osd.4 [v2:172.21.15.134:6800/3588917663,v1:172.21.15.134:6801/3588917663]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]: dispatch 2024-07-15T06:44:04.623 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:04 smithi089 bash[21793]: cluster 2024-07-15T06:44:03.242972+0000 mon.a (mon.0) 456 : cluster [DBG] osdmap e28: 5 total, 4 up, 5 in 2024-07-15T06:44:04.623 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:04 smithi089 bash[21793]: audit 2024-07-15T06:44:03.243225+0000 mon.a (mon.0) 457 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-07-15T06:44:04.623 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:04 smithi089 bash[21793]: audit 2024-07-15T06:44:03.244550+0000 mon.a (mon.0) 458 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]: dispatch 2024-07-15T06:44:05.307 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:05 smithi134 bash[24068]: audit 2024-07-15T06:44:04.247443+0000 mon.a (mon.0) 459 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]': finished 2024-07-15T06:44:05.308 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:05 smithi134 bash[24068]: cluster 2024-07-15T06:44:04.247615+0000 mon.a (mon.0) 460 : cluster [DBG] osdmap e29: 5 total, 4 up, 5 in 2024-07-15T06:44:05.308 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:05 smithi134 bash[24068]: audit 2024-07-15T06:44:04.248355+0000 mon.a (mon.0) 461 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-07-15T06:44:05.308 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:05 smithi134 bash[24068]: audit 2024-07-15T06:44:04.258378+0000 mon.a (mon.0) 462 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-07-15T06:44:05.622 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:05 smithi089 bash[21793]: audit 2024-07-15T06:44:04.247443+0000 mon.a (mon.0) 459 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]': finished 2024-07-15T06:44:05.622 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:05 smithi089 bash[21793]: cluster 2024-07-15T06:44:04.247615+0000 mon.a (mon.0) 460 : cluster [DBG] osdmap e29: 5 total, 4 up, 5 in 2024-07-15T06:44:05.622 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:05 smithi089 bash[21793]: audit 2024-07-15T06:44:04.248355+0000 mon.a (mon.0) 461 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-07-15T06:44:05.622 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:05 smithi089 bash[21793]: audit 2024-07-15T06:44:04.258378+0000 mon.a (mon.0) 462 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-07-15T06:44:05.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:05 smithi089 bash[31372]: audit 2024-07-15T06:44:04.247443+0000 mon.a (mon.0) 459 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]': finished 2024-07-15T06:44:05.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:05 smithi089 bash[31372]: cluster 2024-07-15T06:44:04.247615+0000 mon.a (mon.0) 460 : cluster [DBG] osdmap e29: 5 total, 4 up, 5 in 2024-07-15T06:44:05.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:05 smithi089 bash[31372]: audit 2024-07-15T06:44:04.248355+0000 mon.a (mon.0) 461 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-07-15T06:44:05.623 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:05 smithi089 bash[31372]: audit 2024-07-15T06:44:04.258378+0000 mon.a (mon.0) 462 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-07-15T06:44:06.161 INFO:teuthology.orchestra.run.smithi134.stdout:Created osd(s) 4 on host 'smithi134' 2024-07-15T06:44:06.558 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:06 smithi134 bash[24068]: cluster 2024-07-15T06:44:03.847379+0000 osd.4 (osd.4) 1 : cluster [DBG] purged_snaps scrub starts 2024-07-15T06:44:06.558 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:06 smithi134 bash[24068]: cluster 2024-07-15T06:44:03.847499+0000 osd.4 (osd.4) 2 : cluster [DBG] purged_snaps scrub ok 2024-07-15T06:44:06.558 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:06 smithi134 bash[24068]: cluster 2024-07-15T06:44:04.932475+0000 mgr.y (mgr.14150) 228 : cluster [DBG] pgmap v198: 1 pgs: 1 peering; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-15T06:44:06.558 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:06 smithi134 bash[24068]: cluster 2024-07-15T06:44:05.248903+0000 mon.a (mon.0) 463 : cluster [INF] osd.4 [v2:172.21.15.134:6800/3588917663,v1:172.21.15.134:6801/3588917663] boot 2024-07-15T06:44:06.558 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:06 smithi134 bash[24068]: cluster 2024-07-15T06:44:05.249003+0000 mon.a (mon.0) 464 : cluster [DBG] osdmap e30: 5 total, 5 up, 5 in 2024-07-15T06:44:06.558 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:06 smithi134 bash[24068]: audit 2024-07-15T06:44:05.251038+0000 mon.a (mon.0) 465 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-07-15T06:44:06.558 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:06 smithi134 bash[24068]: audit 2024-07-15T06:44:06.142427+0000 mon.a (mon.0) 466 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:44:06.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:06 smithi134 bash[24068]: audit 2024-07-15T06:44:06.149837+0000 mon.a (mon.0) 467 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:44:06.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:06 smithi134 bash[24068]: audit 2024-07-15T06:44:06.160128+0000 mon.a (mon.0) 468 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:44:06.622 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:06 smithi089 bash[21793]: cluster 2024-07-15T06:44:03.847379+0000 osd.4 (osd.4) 1 : cluster [DBG] purged_snaps scrub starts 2024-07-15T06:44:06.622 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:06 smithi089 bash[21793]: cluster 2024-07-15T06:44:03.847499+0000 osd.4 (osd.4) 2 : cluster [DBG] purged_snaps scrub ok 2024-07-15T06:44:06.622 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:06 smithi089 bash[21793]: cluster 2024-07-15T06:44:04.932475+0000 mgr.y (mgr.14150) 228 : cluster [DBG] pgmap v198: 1 pgs: 1 peering; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-15T06:44:06.622 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:06 smithi089 bash[21793]: cluster 2024-07-15T06:44:05.248903+0000 mon.a (mon.0) 463 : cluster [INF] osd.4 [v2:172.21.15.134:6800/3588917663,v1:172.21.15.134:6801/3588917663] boot 2024-07-15T06:44:06.622 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:06 smithi089 bash[21793]: cluster 2024-07-15T06:44:05.249003+0000 mon.a (mon.0) 464 : cluster [DBG] osdmap e30: 5 total, 5 up, 5 in 2024-07-15T06:44:06.622 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:06 smithi089 bash[21793]: audit 2024-07-15T06:44:05.251038+0000 mon.a (mon.0) 465 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-07-15T06:44:06.622 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:06 smithi089 bash[21793]: audit 2024-07-15T06:44:06.142427+0000 mon.a (mon.0) 466 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:44:06.622 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:06 smithi089 bash[21793]: audit 2024-07-15T06:44:06.149837+0000 mon.a (mon.0) 467 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:44:06.622 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:06 smithi089 bash[21793]: audit 2024-07-15T06:44:06.160128+0000 mon.a (mon.0) 468 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:44:06.623 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:06 smithi089 bash[31372]: cluster 2024-07-15T06:44:03.847379+0000 osd.4 (osd.4) 1 : cluster [DBG] purged_snaps scrub starts 2024-07-15T06:44:06.623 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:06 smithi089 bash[31372]: cluster 2024-07-15T06:44:03.847499+0000 osd.4 (osd.4) 2 : cluster [DBG] purged_snaps scrub ok 2024-07-15T06:44:06.623 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:06 smithi089 bash[31372]: cluster 2024-07-15T06:44:04.932475+0000 mgr.y (mgr.14150) 228 : cluster [DBG] pgmap v198: 1 pgs: 1 peering; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-15T06:44:06.623 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:06 smithi089 bash[31372]: cluster 2024-07-15T06:44:05.248903+0000 mon.a (mon.0) 463 : cluster [INF] osd.4 [v2:172.21.15.134:6800/3588917663,v1:172.21.15.134:6801/3588917663] boot 2024-07-15T06:44:06.623 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:06 smithi089 bash[31372]: cluster 2024-07-15T06:44:05.249003+0000 mon.a (mon.0) 464 : cluster [DBG] osdmap e30: 5 total, 5 up, 5 in 2024-07-15T06:44:06.623 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:06 smithi089 bash[31372]: audit 2024-07-15T06:44:05.251038+0000 mon.a (mon.0) 465 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-07-15T06:44:06.623 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:06 smithi089 bash[31372]: audit 2024-07-15T06:44:06.142427+0000 mon.a (mon.0) 466 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:44:06.623 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:06 smithi089 bash[31372]: audit 2024-07-15T06:44:06.149837+0000 mon.a (mon.0) 467 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:44:06.623 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:06 smithi089 bash[31372]: audit 2024-07-15T06:44:06.160128+0000 mon.a (mon.0) 468 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:44:06.804 DEBUG:teuthology.orchestra.run.smithi134:osd.4> sudo journalctl -f -n 0 -u ceph-75ba02bc-4274-11ef-bcad-c7b262605968@osd.4.service 2024-07-15T06:44:06.807 INFO:tasks.cephadm:Deploying osd.5 on smithi134 with /dev/vg_nvme/lv_3... 2024-07-15T06:44:06.807 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-07-15T06:44:07.557 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:07 smithi134 bash[24068]: cluster 2024-07-15T06:44:06.261389+0000 mon.a (mon.0) 469 : cluster [DBG] osdmap e31: 5 total, 5 up, 5 in 2024-07-15T06:44:07.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:07 smithi089 bash[21793]: cluster 2024-07-15T06:44:06.261389+0000 mon.a (mon.0) 469 : cluster [DBG] osdmap e31: 5 total, 5 up, 5 in 2024-07-15T06:44:07.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:07 smithi089 bash[31372]: cluster 2024-07-15T06:44:06.261389+0000 mon.a (mon.0) 469 : cluster [DBG] osdmap e31: 5 total, 5 up, 5 in 2024-07-15T06:44:08.557 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:08 smithi134 bash[24068]: cluster 2024-07-15T06:44:06.932987+0000 mgr.y (mgr.14150) 229 : cluster [DBG] pgmap v201: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-15T06:44:08.558 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:08 smithi134 bash[24068]: cluster 2024-07-15T06:44:07.285883+0000 mon.a (mon.0) 470 : cluster [DBG] osdmap e32: 5 total, 5 up, 5 in 2024-07-15T06:44:08.622 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:08 smithi089 bash[21793]: cluster 2024-07-15T06:44:06.932987+0000 mgr.y (mgr.14150) 229 : cluster [DBG] pgmap v201: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-15T06:44:08.622 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:08 smithi089 bash[21793]: cluster 2024-07-15T06:44:07.285883+0000 mon.a (mon.0) 470 : cluster [DBG] osdmap e32: 5 total, 5 up, 5 in 2024-07-15T06:44:08.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:08 smithi089 bash[31372]: cluster 2024-07-15T06:44:06.932987+0000 mgr.y (mgr.14150) 229 : cluster [DBG] pgmap v201: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-15T06:44:08.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:08 smithi089 bash[31372]: cluster 2024-07-15T06:44:07.285883+0000 mon.a (mon.0) 470 : cluster [DBG] osdmap e32: 5 total, 5 up, 5 in 2024-07-15T06:44:09.557 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:09 smithi134 bash[24068]: cluster 2024-07-15T06:44:08.287866+0000 mon.a (mon.0) 471 : cluster [DBG] osdmap e33: 5 total, 5 up, 5 in 2024-07-15T06:44:09.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:09 smithi089 bash[31372]: cluster 2024-07-15T06:44:08.287866+0000 mon.a (mon.0) 471 : cluster [DBG] osdmap e33: 5 total, 5 up, 5 in 2024-07-15T06:44:09.622 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:09 smithi089 bash[21793]: cluster 2024-07-15T06:44:08.287866+0000 mon.a (mon.0) 471 : cluster [DBG] osdmap e33: 5 total, 5 up, 5 in 2024-07-15T06:44:10.557 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:10 smithi134 bash[24068]: cluster 2024-07-15T06:44:08.933534+0000 mgr.y (mgr.14150) 230 : cluster [DBG] pgmap v204: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-15T06:44:10.558 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:10 smithi134 bash[24068]: cluster 2024-07-15T06:44:09.289464+0000 mon.a (mon.0) 472 : cluster [DBG] osdmap e34: 5 total, 5 up, 5 in 2024-07-15T06:44:10.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:10 smithi089 bash[21793]: cluster 2024-07-15T06:44:08.933534+0000 mgr.y (mgr.14150) 230 : cluster [DBG] pgmap v204: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-15T06:44:10.622 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:10 smithi089 bash[21793]: cluster 2024-07-15T06:44:09.289464+0000 mon.a (mon.0) 472 : cluster [DBG] osdmap e34: 5 total, 5 up, 5 in 2024-07-15T06:44:10.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:10 smithi089 bash[31372]: cluster 2024-07-15T06:44:08.933534+0000 mgr.y (mgr.14150) 230 : cluster [DBG] pgmap v204: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-15T06:44:10.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:10 smithi089 bash[31372]: cluster 2024-07-15T06:44:09.289464+0000 mon.a (mon.0) 472 : cluster [DBG] osdmap e34: 5 total, 5 up, 5 in 2024-07-15T06:44:11.331 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.b/config 2024-07-15T06:44:12.557 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:12 smithi134 bash[24068]: cluster 2024-07-15T06:44:10.934114+0000 mgr.y (mgr.14150) 231 : cluster [DBG] pgmap v206: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 123 KiB/s, 0 objects/s recovering 2024-07-15T06:44:12.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:12 smithi089 bash[21793]: cluster 2024-07-15T06:44:10.934114+0000 mgr.y (mgr.14150) 231 : cluster [DBG] pgmap v206: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 123 KiB/s, 0 objects/s recovering 2024-07-15T06:44:12.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:12 smithi089 bash[31372]: cluster 2024-07-15T06:44:10.934114+0000 mgr.y (mgr.14150) 231 : cluster [DBG] pgmap v206: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 123 KiB/s, 0 objects/s recovering 2024-07-15T06:44:14.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:14 smithi089 bash[21793]: cluster 2024-07-15T06:44:12.934662+0000 mgr.y (mgr.14150) 232 : cluster [DBG] pgmap v207: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-07-15T06:44:14.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:14 smithi089 bash[31372]: cluster 2024-07-15T06:44:12.934662+0000 mgr.y (mgr.14150) 232 : cluster [DBG] pgmap v207: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-07-15T06:44:14.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:14 smithi134 bash[24068]: cluster 2024-07-15T06:44:12.934662+0000 mgr.y (mgr.14150) 232 : cluster [DBG] pgmap v207: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-07-15T06:44:15.885 INFO:teuthology.orchestra.run.smithi134.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-07-15T06:44:16.050 INFO:teuthology.orchestra.run.smithi134.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-07-15T06:44:16.066 INFO:teuthology.orchestra.run.smithi134.stderr: stderr: 10+0 records in 2024-07-15T06:44:16.066 INFO:teuthology.orchestra.run.smithi134.stderr:10+0 records out 2024-07-15T06:44:16.067 INFO:teuthology.orchestra.run.smithi134.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0141459 s, 741 MB/s 2024-07-15T06:44:16.067 INFO:teuthology.orchestra.run.smithi134.stderr:--> Zapping successful for: 2024-07-15T06:44:16.557 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:16 smithi134 bash[24068]: cluster 2024-07-15T06:44:14.935233+0000 mgr.y (mgr.14150) 233 : cluster [DBG] pgmap v208: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 75 KiB/s, 0 objects/s recovering 2024-07-15T06:44:16.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:16 smithi089 bash[21793]: cluster 2024-07-15T06:44:14.935233+0000 mgr.y (mgr.14150) 233 : cluster [DBG] pgmap v208: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 75 KiB/s, 0 objects/s recovering 2024-07-15T06:44:16.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:16 smithi089 bash[31372]: cluster 2024-07-15T06:44:14.935233+0000 mgr.y (mgr.14150) 233 : cluster [DBG] pgmap v208: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 75 KiB/s, 0 objects/s recovering 2024-07-15T06:44:17.021 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph orch daemon add osd smithi134:vg_nvme/lv_3 2024-07-15T06:44:18.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:18 smithi089 bash[31372]: cluster 2024-07-15T06:44:16.935781+0000 mgr.y (mgr.14150) 234 : cluster [DBG] pgmap v209: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 67 KiB/s, 0 objects/s recovering 2024-07-15T06:44:18.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:18 smithi089 bash[31372]: cephadm 2024-07-15T06:44:17.338284+0000 mgr.y (mgr.14150) 235 : cephadm [INF] Detected new or changed devices on smithi134 2024-07-15T06:44:18.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:18 smithi089 bash[31372]: audit 2024-07-15T06:44:17.346507+0000 mon.a (mon.0) 473 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:44:18.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:18 smithi089 bash[31372]: audit 2024-07-15T06:44:17.354914+0000 mon.a (mon.0) 474 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:44:18.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:18 smithi089 bash[31372]: audit 2024-07-15T06:44:17.356815+0000 mon.a (mon.0) 475 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:44:18.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:18 smithi089 bash[31372]: cephadm 2024-07-15T06:44:17.358071+0000 mgr.y (mgr.14150) 236 : cephadm [INF] Adjusting osd_memory_target on smithi134 to 17265M 2024-07-15T06:44:18.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:18 smithi089 bash[31372]: audit 2024-07-15T06:44:17.366254+0000 mon.a (mon.0) 476 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:44:18.623 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:18 smithi089 bash[31372]: audit 2024-07-15T06:44:17.368203+0000 mon.a (mon.0) 477 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:44:18.623 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:18 smithi089 bash[31372]: audit 2024-07-15T06:44:17.369857+0000 mon.a (mon.0) 478 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:44:18.623 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:18 smithi089 bash[31372]: audit 2024-07-15T06:44:17.379416+0000 mon.a (mon.0) 479 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:44:18.623 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:18 smithi089 bash[21793]: cluster 2024-07-15T06:44:16.935781+0000 mgr.y (mgr.14150) 234 : cluster [DBG] pgmap v209: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 67 KiB/s, 0 objects/s recovering 2024-07-15T06:44:18.623 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:18 smithi089 bash[21793]: cephadm 2024-07-15T06:44:17.338284+0000 mgr.y (mgr.14150) 235 : cephadm [INF] Detected new or changed devices on smithi134 2024-07-15T06:44:18.623 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:18 smithi089 bash[21793]: audit 2024-07-15T06:44:17.346507+0000 mon.a (mon.0) 473 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:44:18.623 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:18 smithi089 bash[21793]: audit 2024-07-15T06:44:17.354914+0000 mon.a (mon.0) 474 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:44:18.623 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:18 smithi089 bash[21793]: audit 2024-07-15T06:44:17.356815+0000 mon.a (mon.0) 475 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:44:18.623 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:18 smithi089 bash[21793]: cephadm 2024-07-15T06:44:17.358071+0000 mgr.y (mgr.14150) 236 : cephadm [INF] Adjusting osd_memory_target on smithi134 to 17265M 2024-07-15T06:44:18.623 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:18 smithi089 bash[21793]: audit 2024-07-15T06:44:17.366254+0000 mon.a (mon.0) 476 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:44:18.623 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:18 smithi089 bash[21793]: audit 2024-07-15T06:44:17.368203+0000 mon.a (mon.0) 477 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:44:18.624 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:18 smithi089 bash[21793]: audit 2024-07-15T06:44:17.369857+0000 mon.a (mon.0) 478 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:44:18.624 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:18 smithi089 bash[21793]: audit 2024-07-15T06:44:17.379416+0000 mon.a (mon.0) 479 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:44:18.808 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:18 smithi134 bash[24068]: cluster 2024-07-15T06:44:16.935781+0000 mgr.y (mgr.14150) 234 : cluster [DBG] pgmap v209: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 67 KiB/s, 0 objects/s recovering 2024-07-15T06:44:18.808 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:18 smithi134 bash[24068]: cephadm 2024-07-15T06:44:17.338284+0000 mgr.y (mgr.14150) 235 : cephadm [INF] Detected new or changed devices on smithi134 2024-07-15T06:44:18.808 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:18 smithi134 bash[24068]: audit 2024-07-15T06:44:17.346507+0000 mon.a (mon.0) 473 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:44:18.808 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:18 smithi134 bash[24068]: audit 2024-07-15T06:44:17.354914+0000 mon.a (mon.0) 474 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:44:18.808 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:18 smithi134 bash[24068]: audit 2024-07-15T06:44:17.356815+0000 mon.a (mon.0) 475 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:44:18.808 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:18 smithi134 bash[24068]: cephadm 2024-07-15T06:44:17.358071+0000 mgr.y (mgr.14150) 236 : cephadm [INF] Adjusting osd_memory_target on smithi134 to 17265M 2024-07-15T06:44:18.808 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:18 smithi134 bash[24068]: audit 2024-07-15T06:44:17.366254+0000 mon.a (mon.0) 476 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:44:18.809 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:18 smithi134 bash[24068]: audit 2024-07-15T06:44:17.368203+0000 mon.a (mon.0) 477 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:44:18.809 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:18 smithi134 bash[24068]: audit 2024-07-15T06:44:17.369857+0000 mon.a (mon.0) 478 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:44:18.809 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:18 smithi134 bash[24068]: audit 2024-07-15T06:44:17.379416+0000 mon.a (mon.0) 479 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:44:20.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:20 smithi089 bash[21793]: cluster 2024-07-15T06:44:18.936315+0000 mgr.y (mgr.14150) 237 : cluster [DBG] pgmap v210: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-07-15T06:44:20.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:20 smithi089 bash[31372]: cluster 2024-07-15T06:44:18.936315+0000 mgr.y (mgr.14150) 237 : cluster [DBG] pgmap v210: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-07-15T06:44:20.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:20 smithi134 bash[24068]: cluster 2024-07-15T06:44:18.936315+0000 mgr.y (mgr.14150) 237 : cluster [DBG] pgmap v210: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-07-15T06:44:21.742 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.b/config 2024-07-15T06:44:22.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:22 smithi089 bash[21793]: cluster 2024-07-15T06:44:20.936887+0000 mgr.y (mgr.14150) 238 : cluster [DBG] pgmap v211: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 49 KiB/s, 0 objects/s recovering 2024-07-15T06:44:22.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:22 smithi089 bash[31372]: cluster 2024-07-15T06:44:20.936887+0000 mgr.y (mgr.14150) 238 : cluster [DBG] pgmap v211: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 49 KiB/s, 0 objects/s recovering 2024-07-15T06:44:22.751 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:22 smithi134 bash[24068]: cluster 2024-07-15T06:44:20.936887+0000 mgr.y (mgr.14150) 238 : cluster [DBG] pgmap v211: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 49 KiB/s, 0 objects/s recovering 2024-07-15T06:44:24.604 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:24 smithi134 bash[24068]: cluster 2024-07-15T06:44:22.937423+0000 mgr.y (mgr.14150) 239 : cluster [DBG] pgmap v212: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-15T06:44:24.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:24 smithi089 bash[21793]: cluster 2024-07-15T06:44:22.937423+0000 mgr.y (mgr.14150) 239 : cluster [DBG] pgmap v212: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-15T06:44:24.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:24 smithi089 bash[31372]: cluster 2024-07-15T06:44:22.937423+0000 mgr.y (mgr.14150) 239 : cluster [DBG] pgmap v212: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-15T06:44:25.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:25 smithi089 bash[21793]: audit 2024-07-15T06:44:24.919925+0000 mon.a (mon.0) 480 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-07-15T06:44:25.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:25 smithi089 bash[21793]: audit 2024-07-15T06:44:24.925828+0000 mon.a (mon.0) 481 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-07-15T06:44:25.622 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:25 smithi089 bash[21793]: audit 2024-07-15T06:44:24.927570+0000 mon.a (mon.0) 482 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:44:25.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:25 smithi089 bash[31372]: audit 2024-07-15T06:44:24.919925+0000 mon.a (mon.0) 480 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-07-15T06:44:25.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:25 smithi089 bash[31372]: audit 2024-07-15T06:44:24.925828+0000 mon.a (mon.0) 481 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-07-15T06:44:25.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:25 smithi089 bash[31372]: audit 2024-07-15T06:44:24.927570+0000 mon.a (mon.0) 482 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:44:25.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:25 smithi134 bash[24068]: audit 2024-07-15T06:44:24.919925+0000 mon.a (mon.0) 480 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-07-15T06:44:25.808 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:25 smithi134 bash[24068]: audit 2024-07-15T06:44:24.925828+0000 mon.a (mon.0) 481 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-07-15T06:44:25.808 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:25 smithi134 bash[24068]: audit 2024-07-15T06:44:24.927570+0000 mon.a (mon.0) 482 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:44:26.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:26 smithi089 bash[21793]: audit 2024-07-15T06:44:24.913533+0000 mgr.y (mgr.14150) 240 : audit [DBG] from='client.24205 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi134:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T06:44:26.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:26 smithi089 bash[21793]: cluster 2024-07-15T06:44:24.937965+0000 mgr.y (mgr.14150) 241 : cluster [DBG] pgmap v213: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-15T06:44:26.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:26 smithi089 bash[31372]: audit 2024-07-15T06:44:24.913533+0000 mgr.y (mgr.14150) 240 : audit [DBG] from='client.24205 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi134:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T06:44:26.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:26 smithi089 bash[31372]: cluster 2024-07-15T06:44:24.937965+0000 mgr.y (mgr.14150) 241 : cluster [DBG] pgmap v213: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-15T06:44:26.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:26 smithi134 bash[24068]: audit 2024-07-15T06:44:24.913533+0000 mgr.y (mgr.14150) 240 : audit [DBG] from='client.24205 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi134:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T06:44:26.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:26 smithi134 bash[24068]: cluster 2024-07-15T06:44:24.937965+0000 mgr.y (mgr.14150) 241 : cluster [DBG] pgmap v213: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-15T06:44:28.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:28 smithi089 bash[21793]: cluster 2024-07-15T06:44:26.938443+0000 mgr.y (mgr.14150) 242 : cluster [DBG] pgmap v214: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-15T06:44:28.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:28 smithi089 bash[31372]: cluster 2024-07-15T06:44:26.938443+0000 mgr.y (mgr.14150) 242 : cluster [DBG] pgmap v214: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-15T06:44:28.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:28 smithi134 bash[24068]: cluster 2024-07-15T06:44:26.938443+0000 mgr.y (mgr.14150) 242 : cluster [DBG] pgmap v214: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-15T06:44:30.631 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:30 smithi134 bash[24068]: cluster 2024-07-15T06:44:28.938992+0000 mgr.y (mgr.14150) 243 : cluster [DBG] pgmap v215: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-15T06:44:30.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:30 smithi089 bash[21793]: cluster 2024-07-15T06:44:28.938992+0000 mgr.y (mgr.14150) 243 : cluster [DBG] pgmap v215: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-15T06:44:30.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:30 smithi089 bash[31372]: cluster 2024-07-15T06:44:28.938992+0000 mgr.y (mgr.14150) 243 : cluster [DBG] pgmap v215: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-15T06:44:32.557 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:32 smithi134 bash[24068]: cluster 2024-07-15T06:44:30.939547+0000 mgr.y (mgr.14150) 244 : cluster [DBG] pgmap v216: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-15T06:44:32.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:32 smithi089 bash[21793]: cluster 2024-07-15T06:44:30.939547+0000 mgr.y (mgr.14150) 244 : cluster [DBG] pgmap v216: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-15T06:44:32.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:32 smithi089 bash[31372]: cluster 2024-07-15T06:44:30.939547+0000 mgr.y (mgr.14150) 244 : cluster [DBG] pgmap v216: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-15T06:44:33.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:33 smithi134 bash[24068]: audit 2024-07-15T06:44:32.963509+0000 mon.b (mon.1) 13 : audit [INF] from='client.? 172.21.15.134:0/2752428360' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a834c8ee-e034-423d-ab67-1ab33bc15a97"}]: dispatch 2024-07-15T06:44:33.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:33 smithi134 bash[24068]: audit 2024-07-15T06:44:32.965599+0000 mon.a (mon.0) 483 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a834c8ee-e034-423d-ab67-1ab33bc15a97"}]: dispatch 2024-07-15T06:44:33.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:33 smithi134 bash[24068]: audit 2024-07-15T06:44:32.974069+0000 mon.a (mon.0) 484 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a834c8ee-e034-423d-ab67-1ab33bc15a97"}]': finished 2024-07-15T06:44:33.808 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:33 smithi134 bash[24068]: cluster 2024-07-15T06:44:32.974178+0000 mon.a (mon.0) 485 : cluster [DBG] osdmap e35: 6 total, 5 up, 6 in 2024-07-15T06:44:33.808 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:33 smithi134 bash[24068]: audit 2024-07-15T06:44:32.974446+0000 mon.a (mon.0) 486 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-07-15T06:44:33.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:33 smithi089 bash[21793]: audit 2024-07-15T06:44:32.963509+0000 mon.b (mon.1) 13 : audit [INF] from='client.? 172.21.15.134:0/2752428360' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a834c8ee-e034-423d-ab67-1ab33bc15a97"}]: dispatch 2024-07-15T06:44:33.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:33 smithi089 bash[21793]: audit 2024-07-15T06:44:32.965599+0000 mon.a (mon.0) 483 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a834c8ee-e034-423d-ab67-1ab33bc15a97"}]: dispatch 2024-07-15T06:44:33.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:33 smithi089 bash[21793]: audit 2024-07-15T06:44:32.974069+0000 mon.a (mon.0) 484 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a834c8ee-e034-423d-ab67-1ab33bc15a97"}]': finished 2024-07-15T06:44:33.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:33 smithi089 bash[21793]: cluster 2024-07-15T06:44:32.974178+0000 mon.a (mon.0) 485 : cluster [DBG] osdmap e35: 6 total, 5 up, 6 in 2024-07-15T06:44:33.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:33 smithi089 bash[21793]: audit 2024-07-15T06:44:32.974446+0000 mon.a (mon.0) 486 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-07-15T06:44:33.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:33 smithi089 bash[31372]: audit 2024-07-15T06:44:32.963509+0000 mon.b (mon.1) 13 : audit [INF] from='client.? 172.21.15.134:0/2752428360' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a834c8ee-e034-423d-ab67-1ab33bc15a97"}]: dispatch 2024-07-15T06:44:33.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:33 smithi089 bash[31372]: audit 2024-07-15T06:44:32.965599+0000 mon.a (mon.0) 483 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a834c8ee-e034-423d-ab67-1ab33bc15a97"}]: dispatch 2024-07-15T06:44:33.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:33 smithi089 bash[31372]: audit 2024-07-15T06:44:32.974069+0000 mon.a (mon.0) 484 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a834c8ee-e034-423d-ab67-1ab33bc15a97"}]': finished 2024-07-15T06:44:33.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:33 smithi089 bash[31372]: cluster 2024-07-15T06:44:32.974178+0000 mon.a (mon.0) 485 : cluster [DBG] osdmap e35: 6 total, 5 up, 6 in 2024-07-15T06:44:33.873 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:33 smithi089 bash[31372]: audit 2024-07-15T06:44:32.974446+0000 mon.a (mon.0) 486 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-07-15T06:44:34.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:34 smithi134 bash[24068]: cluster 2024-07-15T06:44:32.940054+0000 mgr.y (mgr.14150) 245 : cluster [DBG] pgmap v217: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-15T06:44:34.808 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:34 smithi134 bash[24068]: audit 2024-07-15T06:44:33.615468+0000 mon.b (mon.1) 14 : audit [DBG] from='client.? 172.21.15.134:0/4064453100' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-07-15T06:44:34.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:34 smithi089 bash[31372]: cluster 2024-07-15T06:44:32.940054+0000 mgr.y (mgr.14150) 245 : cluster [DBG] pgmap v217: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-15T06:44:34.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:34 smithi089 bash[31372]: audit 2024-07-15T06:44:33.615468+0000 mon.b (mon.1) 14 : audit [DBG] from='client.? 172.21.15.134:0/4064453100' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-07-15T06:44:34.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:34 smithi089 bash[21793]: cluster 2024-07-15T06:44:32.940054+0000 mgr.y (mgr.14150) 245 : cluster [DBG] pgmap v217: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-15T06:44:34.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:34 smithi089 bash[21793]: audit 2024-07-15T06:44:33.615468+0000 mon.b (mon.1) 14 : audit [DBG] from='client.? 172.21.15.134:0/4064453100' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-07-15T06:44:36.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:36 smithi134 bash[24068]: cluster 2024-07-15T06:44:34.940609+0000 mgr.y (mgr.14150) 246 : cluster [DBG] pgmap v219: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-15T06:44:36.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:36 smithi089 bash[21793]: cluster 2024-07-15T06:44:34.940609+0000 mgr.y (mgr.14150) 246 : cluster [DBG] pgmap v219: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-15T06:44:36.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:36 smithi089 bash[31372]: cluster 2024-07-15T06:44:34.940609+0000 mgr.y (mgr.14150) 246 : cluster [DBG] pgmap v219: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-15T06:44:38.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:38 smithi134 bash[24068]: cluster 2024-07-15T06:44:36.941128+0000 mgr.y (mgr.14150) 247 : cluster [DBG] pgmap v220: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-15T06:44:38.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:38 smithi089 bash[21793]: cluster 2024-07-15T06:44:36.941128+0000 mgr.y (mgr.14150) 247 : cluster [DBG] pgmap v220: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-15T06:44:38.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:38 smithi089 bash[31372]: cluster 2024-07-15T06:44:36.941128+0000 mgr.y (mgr.14150) 247 : cluster [DBG] pgmap v220: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-15T06:44:40.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:40 smithi134 bash[24068]: cluster 2024-07-15T06:44:38.941643+0000 mgr.y (mgr.14150) 248 : cluster [DBG] pgmap v221: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-15T06:44:40.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:40 smithi089 bash[31372]: cluster 2024-07-15T06:44:38.941643+0000 mgr.y (mgr.14150) 248 : cluster [DBG] pgmap v221: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-15T06:44:40.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:40 smithi089 bash[21793]: cluster 2024-07-15T06:44:38.941643+0000 mgr.y (mgr.14150) 248 : cluster [DBG] pgmap v221: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-15T06:44:42.672 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:42 smithi134 bash[24068]: cluster 2024-07-15T06:44:40.942224+0000 mgr.y (mgr.14150) 249 : cluster [DBG] pgmap v222: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-15T06:44:42.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:42 smithi089 bash[31372]: cluster 2024-07-15T06:44:40.942224+0000 mgr.y (mgr.14150) 249 : cluster [DBG] pgmap v222: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-15T06:44:42.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:42 smithi089 bash[21793]: cluster 2024-07-15T06:44:40.942224+0000 mgr.y (mgr.14150) 249 : cluster [DBG] pgmap v222: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-15T06:44:44.638 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:44 smithi134 bash[24068]: cluster 2024-07-15T06:44:42.942774+0000 mgr.y (mgr.14150) 250 : cluster [DBG] pgmap v223: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-15T06:44:44.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:44 smithi089 bash[31372]: cluster 2024-07-15T06:44:42.942774+0000 mgr.y (mgr.14150) 250 : cluster [DBG] pgmap v223: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-15T06:44:44.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:44 smithi089 bash[21793]: cluster 2024-07-15T06:44:42.942774+0000 mgr.y (mgr.14150) 250 : cluster [DBG] pgmap v223: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-15T06:44:45.557 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:45 smithi134 bash[24068]: audit 2024-07-15T06:44:45.036415+0000 mon.a (mon.0) 487 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-07-15T06:44:45.557 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:45 smithi134 bash[24068]: audit 2024-07-15T06:44:45.038140+0000 mon.a (mon.0) 488 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:44:45.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:45 smithi089 bash[21793]: audit 2024-07-15T06:44:45.036415+0000 mon.a (mon.0) 487 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-07-15T06:44:45.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:45 smithi089 bash[21793]: audit 2024-07-15T06:44:45.038140+0000 mon.a (mon.0) 488 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:44:45.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:45 smithi089 bash[31372]: audit 2024-07-15T06:44:45.036415+0000 mon.a (mon.0) 487 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-07-15T06:44:45.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:45 smithi089 bash[31372]: audit 2024-07-15T06:44:45.038140+0000 mon.a (mon.0) 488 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:44:46.557 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:46 smithi134 bash[24068]: cluster 2024-07-15T06:44:44.943388+0000 mgr.y (mgr.14150) 251 : cluster [DBG] pgmap v224: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-15T06:44:46.557 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:46 smithi134 bash[24068]: cephadm 2024-07-15T06:44:45.039615+0000 mgr.y (mgr.14150) 252 : cephadm [INF] Deploying daemon osd.5 on smithi134 2024-07-15T06:44:46.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:46 smithi089 bash[31372]: cluster 2024-07-15T06:44:44.943388+0000 mgr.y (mgr.14150) 251 : cluster [DBG] pgmap v224: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-15T06:44:46.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:46 smithi089 bash[31372]: cephadm 2024-07-15T06:44:45.039615+0000 mgr.y (mgr.14150) 252 : cephadm [INF] Deploying daemon osd.5 on smithi134 2024-07-15T06:44:46.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:46 smithi089 bash[21793]: cluster 2024-07-15T06:44:44.943388+0000 mgr.y (mgr.14150) 251 : cluster [DBG] pgmap v224: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-15T06:44:46.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:46 smithi089 bash[21793]: cephadm 2024-07-15T06:44:45.039615+0000 mgr.y (mgr.14150) 252 : cephadm [INF] Deploying daemon osd.5 on smithi134 2024-07-15T06:44:48.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:48 smithi134 bash[24068]: cluster 2024-07-15T06:44:46.943940+0000 mgr.y (mgr.14150) 253 : cluster [DBG] pgmap v225: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-15T06:44:48.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:48 smithi089 bash[31372]: cluster 2024-07-15T06:44:46.943940+0000 mgr.y (mgr.14150) 253 : cluster [DBG] pgmap v225: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-15T06:44:48.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:48 smithi089 bash[21793]: cluster 2024-07-15T06:44:46.943940+0000 mgr.y (mgr.14150) 253 : cluster [DBG] pgmap v225: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-15T06:44:49.152 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:44:49 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:44:49.152 INFO:journalctl@ceph.osd.4.smithi134.stdout:Jul 15 06:44:49 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:44:49.152 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:49 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:44:49.406 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:49 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:44:49.406 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:44:49 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:44:49.406 INFO:journalctl@ceph.osd.4.smithi134.stdout:Jul 15 06:44:49 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:44:49.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:49 smithi134 bash[24068]: audit 2024-07-15T06:44:49.440601+0000 mon.a (mon.0) 489 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:44:49.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:49 smithi134 bash[24068]: audit 2024-07-15T06:44:49.452693+0000 mon.a (mon.0) 490 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:44:49.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:49 smithi134 bash[24068]: audit 2024-07-15T06:44:49.460701+0000 mon.a (mon.0) 491 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:44:49.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:49 smithi089 bash[31372]: audit 2024-07-15T06:44:49.440601+0000 mon.a (mon.0) 489 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:44:49.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:49 smithi089 bash[31372]: audit 2024-07-15T06:44:49.452693+0000 mon.a (mon.0) 490 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:44:49.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:49 smithi089 bash[31372]: audit 2024-07-15T06:44:49.460701+0000 mon.a (mon.0) 491 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:44:49.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:49 smithi089 bash[21793]: audit 2024-07-15T06:44:49.440601+0000 mon.a (mon.0) 489 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:44:49.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:49 smithi089 bash[21793]: audit 2024-07-15T06:44:49.452693+0000 mon.a (mon.0) 490 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:44:49.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:49 smithi089 bash[21793]: audit 2024-07-15T06:44:49.460701+0000 mon.a (mon.0) 491 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:44:50.773 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:50 smithi134 bash[24068]: cluster 2024-07-15T06:44:48.944602+0000 mgr.y (mgr.14150) 254 : cluster [DBG] pgmap v226: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-15T06:44:50.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:50 smithi089 bash[21793]: cluster 2024-07-15T06:44:48.944602+0000 mgr.y (mgr.14150) 254 : cluster [DBG] pgmap v226: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-15T06:44:50.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:50 smithi089 bash[31372]: cluster 2024-07-15T06:44:48.944602+0000 mgr.y (mgr.14150) 254 : cluster [DBG] pgmap v226: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-15T06:44:52.557 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:52 smithi134 bash[24068]: cluster 2024-07-15T06:44:50.945179+0000 mgr.y (mgr.14150) 255 : cluster [DBG] pgmap v227: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-15T06:44:52.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:52 smithi089 bash[21793]: cluster 2024-07-15T06:44:50.945179+0000 mgr.y (mgr.14150) 255 : cluster [DBG] pgmap v227: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-15T06:44:52.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:52 smithi089 bash[31372]: cluster 2024-07-15T06:44:50.945179+0000 mgr.y (mgr.14150) 255 : cluster [DBG] pgmap v227: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-15T06:44:53.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:53 smithi134 bash[24068]: cluster 2024-07-15T06:44:52.945630+0000 mgr.y (mgr.14150) 256 : cluster [DBG] pgmap v228: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-15T06:44:53.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:53 smithi089 bash[21793]: cluster 2024-07-15T06:44:52.945630+0000 mgr.y (mgr.14150) 256 : cluster [DBG] pgmap v228: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-15T06:44:53.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:53 smithi089 bash[31372]: cluster 2024-07-15T06:44:52.945630+0000 mgr.y (mgr.14150) 256 : cluster [DBG] pgmap v228: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-15T06:44:55.557 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:55 smithi134 bash[24068]: audit 2024-07-15T06:44:54.517228+0000 mon.a (mon.0) 492 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:44:55.557 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:55 smithi134 bash[24068]: audit 2024-07-15T06:44:54.527520+0000 mon.a (mon.0) 493 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:44:55.557 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:55 smithi134 bash[24068]: audit 2024-07-15T06:44:54.529337+0000 mon.a (mon.0) 494 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:44:55.558 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:55 smithi134 bash[24068]: audit 2024-07-15T06:44:54.531127+0000 mon.a (mon.0) 495 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:44:55.558 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:55 smithi134 bash[24068]: audit 2024-07-15T06:44:54.542489+0000 mon.a (mon.0) 496 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:44:55.558 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:55 smithi134 bash[24068]: cluster 2024-07-15T06:44:54.946175+0000 mgr.y (mgr.14150) 257 : cluster [DBG] pgmap v229: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-15T06:44:55.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:55 smithi089 bash[21793]: audit 2024-07-15T06:44:54.517228+0000 mon.a (mon.0) 492 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:44:55.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:55 smithi089 bash[21793]: audit 2024-07-15T06:44:54.527520+0000 mon.a (mon.0) 493 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:44:55.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:55 smithi089 bash[21793]: audit 2024-07-15T06:44:54.529337+0000 mon.a (mon.0) 494 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:44:55.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:55 smithi089 bash[21793]: audit 2024-07-15T06:44:54.531127+0000 mon.a (mon.0) 495 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:44:55.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:55 smithi089 bash[21793]: audit 2024-07-15T06:44:54.542489+0000 mon.a (mon.0) 496 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:44:55.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:55 smithi089 bash[21793]: cluster 2024-07-15T06:44:54.946175+0000 mgr.y (mgr.14150) 257 : cluster [DBG] pgmap v229: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-15T06:44:55.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:55 smithi089 bash[31372]: audit 2024-07-15T06:44:54.517228+0000 mon.a (mon.0) 492 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:44:55.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:55 smithi089 bash[31372]: audit 2024-07-15T06:44:54.527520+0000 mon.a (mon.0) 493 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:44:55.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:55 smithi089 bash[31372]: audit 2024-07-15T06:44:54.529337+0000 mon.a (mon.0) 494 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:44:55.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:55 smithi089 bash[31372]: audit 2024-07-15T06:44:54.531127+0000 mon.a (mon.0) 495 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:44:55.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:55 smithi089 bash[31372]: audit 2024-07-15T06:44:54.542489+0000 mon.a (mon.0) 496 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:44:55.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:55 smithi089 bash[31372]: cluster 2024-07-15T06:44:54.946175+0000 mgr.y (mgr.14150) 257 : cluster [DBG] pgmap v229: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-15T06:44:56.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:56 smithi134 bash[24068]: audit 2024-07-15T06:44:55.856097+0000 mon.b (mon.1) 15 : audit [INF] from='osd.5 [v2:172.21.15.134:6808/823577783,v1:172.21.15.134:6809/823577783]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-07-15T06:44:56.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:56 smithi134 bash[24068]: audit 2024-07-15T06:44:55.857924+0000 mon.a (mon.0) 497 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-07-15T06:44:56.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:56 smithi089 bash[21793]: audit 2024-07-15T06:44:55.856097+0000 mon.b (mon.1) 15 : audit [INF] from='osd.5 [v2:172.21.15.134:6808/823577783,v1:172.21.15.134:6809/823577783]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-07-15T06:44:56.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:56 smithi089 bash[21793]: audit 2024-07-15T06:44:55.857924+0000 mon.a (mon.0) 497 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-07-15T06:44:56.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:56 smithi089 bash[31372]: audit 2024-07-15T06:44:55.856097+0000 mon.b (mon.1) 15 : audit [INF] from='osd.5 [v2:172.21.15.134:6808/823577783,v1:172.21.15.134:6809/823577783]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-07-15T06:44:56.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:56 smithi089 bash[31372]: audit 2024-07-15T06:44:55.857924+0000 mon.a (mon.0) 497 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-07-15T06:44:57.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:57 smithi134 bash[24068]: audit 2024-07-15T06:44:56.553278+0000 mon.a (mon.0) 498 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-07-15T06:44:57.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:57 smithi134 bash[24068]: audit 2024-07-15T06:44:56.553430+0000 mon.b (mon.1) 16 : audit [INF] from='osd.5 [v2:172.21.15.134:6808/823577783,v1:172.21.15.134:6809/823577783]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]: dispatch 2024-07-15T06:44:57.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:57 smithi134 bash[24068]: cluster 2024-07-15T06:44:56.553486+0000 mon.a (mon.0) 499 : cluster [DBG] osdmap e36: 6 total, 5 up, 6 in 2024-07-15T06:44:57.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:57 smithi134 bash[24068]: audit 2024-07-15T06:44:56.553874+0000 mon.a (mon.0) 500 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-07-15T06:44:57.808 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:57 smithi134 bash[24068]: audit 2024-07-15T06:44:56.555339+0000 mon.a (mon.0) 501 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]: dispatch 2024-07-15T06:44:57.808 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:57 smithi134 bash[24068]: cluster 2024-07-15T06:44:56.946698+0000 mgr.y (mgr.14150) 258 : cluster [DBG] pgmap v231: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-15T06:44:57.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:57 smithi089 bash[21793]: audit 2024-07-15T06:44:56.553278+0000 mon.a (mon.0) 498 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-07-15T06:44:57.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:57 smithi089 bash[21793]: audit 2024-07-15T06:44:56.553430+0000 mon.b (mon.1) 16 : audit [INF] from='osd.5 [v2:172.21.15.134:6808/823577783,v1:172.21.15.134:6809/823577783]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]: dispatch 2024-07-15T06:44:57.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:57 smithi089 bash[21793]: cluster 2024-07-15T06:44:56.553486+0000 mon.a (mon.0) 499 : cluster [DBG] osdmap e36: 6 total, 5 up, 6 in 2024-07-15T06:44:57.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:57 smithi089 bash[21793]: audit 2024-07-15T06:44:56.553874+0000 mon.a (mon.0) 500 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-07-15T06:44:57.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:57 smithi089 bash[21793]: audit 2024-07-15T06:44:56.555339+0000 mon.a (mon.0) 501 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]: dispatch 2024-07-15T06:44:57.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:57 smithi089 bash[21793]: cluster 2024-07-15T06:44:56.946698+0000 mgr.y (mgr.14150) 258 : cluster [DBG] pgmap v231: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-15T06:44:57.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:57 smithi089 bash[31372]: audit 2024-07-15T06:44:56.553278+0000 mon.a (mon.0) 498 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-07-15T06:44:57.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:57 smithi089 bash[31372]: audit 2024-07-15T06:44:56.553430+0000 mon.b (mon.1) 16 : audit [INF] from='osd.5 [v2:172.21.15.134:6808/823577783,v1:172.21.15.134:6809/823577783]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]: dispatch 2024-07-15T06:44:57.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:57 smithi089 bash[31372]: cluster 2024-07-15T06:44:56.553486+0000 mon.a (mon.0) 499 : cluster [DBG] osdmap e36: 6 total, 5 up, 6 in 2024-07-15T06:44:57.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:57 smithi089 bash[31372]: audit 2024-07-15T06:44:56.553874+0000 mon.a (mon.0) 500 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-07-15T06:44:57.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:57 smithi089 bash[31372]: audit 2024-07-15T06:44:56.555339+0000 mon.a (mon.0) 501 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]: dispatch 2024-07-15T06:44:57.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:57 smithi089 bash[31372]: cluster 2024-07-15T06:44:56.946698+0000 mgr.y (mgr.14150) 258 : cluster [DBG] pgmap v231: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-15T06:44:58.370 INFO:teuthology.orchestra.run.smithi134.stdout:Created osd(s) 5 on host 'smithi134' 2024-07-15T06:44:58.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:58 smithi134 bash[24068]: audit 2024-07-15T06:44:57.556235+0000 mon.a (mon.0) 502 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]': finished 2024-07-15T06:44:58.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:58 smithi134 bash[24068]: cluster 2024-07-15T06:44:57.556479+0000 mon.a (mon.0) 503 : cluster [DBG] osdmap e37: 6 total, 5 up, 6 in 2024-07-15T06:44:58.808 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:58 smithi134 bash[24068]: audit 2024-07-15T06:44:57.557228+0000 mon.a (mon.0) 504 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-07-15T06:44:58.808 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:58 smithi134 bash[24068]: audit 2024-07-15T06:44:57.568038+0000 mon.a (mon.0) 505 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-07-15T06:44:58.808 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:58 smithi134 bash[24068]: audit 2024-07-15T06:44:58.348287+0000 mon.a (mon.0) 506 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:44:58.808 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:58 smithi134 bash[24068]: audit 2024-07-15T06:44:58.357644+0000 mon.a (mon.0) 507 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:44:58.808 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:58 smithi134 bash[24068]: audit 2024-07-15T06:44:58.366546+0000 mon.a (mon.0) 508 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:44:58.808 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:58 smithi134 bash[24068]: cluster 2024-07-15T06:44:58.557389+0000 mon.a (mon.0) 509 : cluster [INF] osd.5 [v2:172.21.15.134:6808/823577783,v1:172.21.15.134:6809/823577783] boot 2024-07-15T06:44:58.808 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:58 smithi134 bash[24068]: cluster 2024-07-15T06:44:58.557491+0000 mon.a (mon.0) 510 : cluster [DBG] osdmap e38: 6 total, 6 up, 6 in 2024-07-15T06:44:58.808 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:58 smithi134 bash[24068]: audit 2024-07-15T06:44:58.557807+0000 mon.a (mon.0) 511 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-07-15T06:44:58.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:58 smithi089 bash[31372]: audit 2024-07-15T06:44:57.556235+0000 mon.a (mon.0) 502 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]': finished 2024-07-15T06:44:58.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:58 smithi089 bash[31372]: cluster 2024-07-15T06:44:57.556479+0000 mon.a (mon.0) 503 : cluster [DBG] osdmap e37: 6 total, 5 up, 6 in 2024-07-15T06:44:58.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:58 smithi089 bash[31372]: audit 2024-07-15T06:44:57.557228+0000 mon.a (mon.0) 504 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-07-15T06:44:58.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:58 smithi089 bash[31372]: audit 2024-07-15T06:44:57.568038+0000 mon.a (mon.0) 505 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-07-15T06:44:58.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:58 smithi089 bash[31372]: audit 2024-07-15T06:44:58.348287+0000 mon.a (mon.0) 506 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:44:58.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:58 smithi089 bash[31372]: audit 2024-07-15T06:44:58.357644+0000 mon.a (mon.0) 507 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:44:58.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:58 smithi089 bash[31372]: audit 2024-07-15T06:44:58.366546+0000 mon.a (mon.0) 508 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:44:58.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:58 smithi089 bash[31372]: cluster 2024-07-15T06:44:58.557389+0000 mon.a (mon.0) 509 : cluster [INF] osd.5 [v2:172.21.15.134:6808/823577783,v1:172.21.15.134:6809/823577783] boot 2024-07-15T06:44:58.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:58 smithi089 bash[31372]: cluster 2024-07-15T06:44:58.557491+0000 mon.a (mon.0) 510 : cluster [DBG] osdmap e38: 6 total, 6 up, 6 in 2024-07-15T06:44:58.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:58 smithi089 bash[31372]: audit 2024-07-15T06:44:58.557807+0000 mon.a (mon.0) 511 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-07-15T06:44:58.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:58 smithi089 bash[21793]: audit 2024-07-15T06:44:57.556235+0000 mon.a (mon.0) 502 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]': finished 2024-07-15T06:44:58.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:58 smithi089 bash[21793]: cluster 2024-07-15T06:44:57.556479+0000 mon.a (mon.0) 503 : cluster [DBG] osdmap e37: 6 total, 5 up, 6 in 2024-07-15T06:44:58.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:58 smithi089 bash[21793]: audit 2024-07-15T06:44:57.557228+0000 mon.a (mon.0) 504 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-07-15T06:44:58.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:58 smithi089 bash[21793]: audit 2024-07-15T06:44:57.568038+0000 mon.a (mon.0) 505 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-07-15T06:44:58.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:58 smithi089 bash[21793]: audit 2024-07-15T06:44:58.348287+0000 mon.a (mon.0) 506 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:44:58.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:58 smithi089 bash[21793]: audit 2024-07-15T06:44:58.357644+0000 mon.a (mon.0) 507 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:44:58.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:58 smithi089 bash[21793]: audit 2024-07-15T06:44:58.366546+0000 mon.a (mon.0) 508 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:44:58.873 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:58 smithi089 bash[21793]: cluster 2024-07-15T06:44:58.557389+0000 mon.a (mon.0) 509 : cluster [INF] osd.5 [v2:172.21.15.134:6808/823577783,v1:172.21.15.134:6809/823577783] boot 2024-07-15T06:44:58.873 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:58 smithi089 bash[21793]: cluster 2024-07-15T06:44:58.557491+0000 mon.a (mon.0) 510 : cluster [DBG] osdmap e38: 6 total, 6 up, 6 in 2024-07-15T06:44:58.873 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:58 smithi089 bash[21793]: audit 2024-07-15T06:44:58.557807+0000 mon.a (mon.0) 511 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-07-15T06:44:59.046 DEBUG:teuthology.orchestra.run.smithi134:osd.5> sudo journalctl -f -n 0 -u ceph-75ba02bc-4274-11ef-bcad-c7b262605968@osd.5.service 2024-07-15T06:44:59.049 INFO:tasks.cephadm:Deploying osd.6 on smithi134 with /dev/vg_nvme/lv_2... 2024-07-15T06:44:59.049 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-07-15T06:44:59.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:59 smithi089 bash[21793]: cluster 2024-07-15T06:44:56.895190+0000 osd.5 (osd.5) 1 : cluster [DBG] purged_snaps scrub starts 2024-07-15T06:44:59.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:59 smithi089 bash[21793]: cluster 2024-07-15T06:44:56.895299+0000 osd.5 (osd.5) 2 : cluster [DBG] purged_snaps scrub ok 2024-07-15T06:44:59.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:59 smithi089 bash[21793]: cluster 2024-07-15T06:44:58.947248+0000 mgr.y (mgr.14150) 259 : cluster [DBG] pgmap v234: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-15T06:44:59.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:44:59 smithi089 bash[21793]: cluster 2024-07-15T06:44:59.560297+0000 mon.a (mon.0) 512 : cluster [DBG] osdmap e39: 6 total, 6 up, 6 in 2024-07-15T06:44:59.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:59 smithi089 bash[31372]: cluster 2024-07-15T06:44:56.895190+0000 osd.5 (osd.5) 1 : cluster [DBG] purged_snaps scrub starts 2024-07-15T06:44:59.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:59 smithi089 bash[31372]: cluster 2024-07-15T06:44:56.895299+0000 osd.5 (osd.5) 2 : cluster [DBG] purged_snaps scrub ok 2024-07-15T06:44:59.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:59 smithi089 bash[31372]: cluster 2024-07-15T06:44:58.947248+0000 mgr.y (mgr.14150) 259 : cluster [DBG] pgmap v234: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-15T06:44:59.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:44:59 smithi089 bash[31372]: cluster 2024-07-15T06:44:59.560297+0000 mon.a (mon.0) 512 : cluster [DBG] osdmap e39: 6 total, 6 up, 6 in 2024-07-15T06:45:00.057 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:59 smithi134 bash[24068]: cluster 2024-07-15T06:44:56.895190+0000 osd.5 (osd.5) 1 : cluster [DBG] purged_snaps scrub starts 2024-07-15T06:45:00.057 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:59 smithi134 bash[24068]: cluster 2024-07-15T06:44:56.895299+0000 osd.5 (osd.5) 2 : cluster [DBG] purged_snaps scrub ok 2024-07-15T06:45:00.057 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:59 smithi134 bash[24068]: cluster 2024-07-15T06:44:58.947248+0000 mgr.y (mgr.14150) 259 : cluster [DBG] pgmap v234: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-15T06:45:00.057 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:44:59 smithi134 bash[24068]: cluster 2024-07-15T06:44:59.560297+0000 mon.a (mon.0) 512 : cluster [DBG] osdmap e39: 6 total, 6 up, 6 in 2024-07-15T06:45:01.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:45:01 smithi089 bash[31372]: cluster 2024-07-15T06:45:00.564312+0000 mon.a (mon.0) 513 : cluster [DBG] osdmap e40: 6 total, 6 up, 6 in 2024-07-15T06:45:01.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:45:01 smithi089 bash[31372]: cluster 2024-07-15T06:45:00.947884+0000 mgr.y (mgr.14150) 260 : cluster [DBG] pgmap v237: 1 pgs: 1 remapped+peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-15T06:45:01.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:45:01 smithi089 bash[31372]: cluster 2024-07-15T06:45:01.308237+0000 mon.a (mon.0) 514 : cluster [DBG] osdmap e41: 6 total, 6 up, 6 in 2024-07-15T06:45:01.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:45:01 smithi089 bash[21793]: cluster 2024-07-15T06:45:00.564312+0000 mon.a (mon.0) 513 : cluster [DBG] osdmap e40: 6 total, 6 up, 6 in 2024-07-15T06:45:01.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:45:01 smithi089 bash[21793]: cluster 2024-07-15T06:45:00.947884+0000 mgr.y (mgr.14150) 260 : cluster [DBG] pgmap v237: 1 pgs: 1 remapped+peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-15T06:45:01.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:45:01 smithi089 bash[21793]: cluster 2024-07-15T06:45:01.308237+0000 mon.a (mon.0) 514 : cluster [DBG] osdmap e41: 6 total, 6 up, 6 in 2024-07-15T06:45:02.057 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:45:01 smithi134 bash[24068]: cluster 2024-07-15T06:45:00.564312+0000 mon.a (mon.0) 513 : cluster [DBG] osdmap e40: 6 total, 6 up, 6 in 2024-07-15T06:45:02.057 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:45:01 smithi134 bash[24068]: cluster 2024-07-15T06:45:00.947884+0000 mgr.y (mgr.14150) 260 : cluster [DBG] pgmap v237: 1 pgs: 1 remapped+peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-15T06:45:02.057 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:45:01 smithi134 bash[24068]: cluster 2024-07-15T06:45:01.308237+0000 mon.a (mon.0) 514 : cluster [DBG] osdmap e41: 6 total, 6 up, 6 in 2024-07-15T06:45:02.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:45:02 smithi089 bash[21793]: cluster 2024-07-15T06:45:01.557318+0000 mon.a (mon.0) 515 : cluster [WRN] Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-07-15T06:45:02.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:45:02 smithi089 bash[21793]: cluster 2024-07-15T06:45:02.311133+0000 mon.a (mon.0) 516 : cluster [DBG] osdmap e42: 6 total, 6 up, 6 in 2024-07-15T06:45:02.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:45:02 smithi089 bash[31372]: cluster 2024-07-15T06:45:01.557318+0000 mon.a (mon.0) 515 : cluster [WRN] Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-07-15T06:45:02.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:45:02 smithi089 bash[31372]: cluster 2024-07-15T06:45:02.311133+0000 mon.a (mon.0) 516 : cluster [DBG] osdmap e42: 6 total, 6 up, 6 in 2024-07-15T06:45:03.057 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:45:02 smithi134 bash[24068]: cluster 2024-07-15T06:45:01.557318+0000 mon.a (mon.0) 515 : cluster [WRN] Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-07-15T06:45:03.057 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:45:02 smithi134 bash[24068]: cluster 2024-07-15T06:45:02.311133+0000 mon.a (mon.0) 516 : cluster [DBG] osdmap e42: 6 total, 6 up, 6 in 2024-07-15T06:45:03.160 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.b/config 2024-07-15T06:45:03.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:45:03 smithi089 bash[21793]: cluster 2024-07-15T06:45:02.948439+0000 mgr.y (mgr.14150) 261 : cluster [DBG] pgmap v240: 1 pgs: 1 remapped+peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-15T06:45:03.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:45:03 smithi089 bash[31372]: cluster 2024-07-15T06:45:02.948439+0000 mgr.y (mgr.14150) 261 : cluster [DBG] pgmap v240: 1 pgs: 1 remapped+peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-15T06:45:04.057 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:45:03 smithi134 bash[24068]: cluster 2024-07-15T06:45:02.948439+0000 mgr.y (mgr.14150) 261 : cluster [DBG] pgmap v240: 1 pgs: 1 remapped+peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-15T06:45:05.301 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:45:05 smithi134 bash[24068]: cluster 2024-07-15T06:45:05.000065+0000 mon.a (mon.0) 517 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-07-15T06:45:05.303 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:45:05 smithi134 bash[24068]: cluster 2024-07-15T06:45:05.000111+0000 mon.a (mon.0) 518 : cluster [INF] Cluster is now healthy 2024-07-15T06:45:05.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:45:05 smithi089 bash[21793]: cluster 2024-07-15T06:45:05.000065+0000 mon.a (mon.0) 517 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-07-15T06:45:05.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:45:05 smithi089 bash[21793]: cluster 2024-07-15T06:45:05.000111+0000 mon.a (mon.0) 518 : cluster [INF] Cluster is now healthy 2024-07-15T06:45:05.371 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:45:05 smithi089 bash[31372]: cluster 2024-07-15T06:45:05.000065+0000 mon.a (mon.0) 517 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-07-15T06:45:05.371 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:45:05 smithi089 bash[31372]: cluster 2024-07-15T06:45:05.000111+0000 mon.a (mon.0) 518 : cluster [INF] Cluster is now healthy 2024-07-15T06:45:06.307 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:45:06 smithi134 bash[24068]: cluster 2024-07-15T06:45:04.948987+0000 mgr.y (mgr.14150) 262 : cluster [DBG] pgmap v241: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 107 KiB/s, 0 objects/s recovering 2024-07-15T06:45:06.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:45:06 smithi089 bash[21793]: cluster 2024-07-15T06:45:04.948987+0000 mgr.y (mgr.14150) 262 : cluster [DBG] pgmap v241: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 107 KiB/s, 0 objects/s recovering 2024-07-15T06:45:06.371 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:45:06 smithi089 bash[31372]: cluster 2024-07-15T06:45:04.948987+0000 mgr.y (mgr.14150) 262 : cluster [DBG] pgmap v241: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 107 KiB/s, 0 objects/s recovering 2024-07-15T06:45:07.915 INFO:teuthology.orchestra.run.smithi134.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-07-15T06:45:08.123 INFO:teuthology.orchestra.run.smithi134.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-07-15T06:45:08.138 INFO:teuthology.orchestra.run.smithi134.stderr: stderr: 10+0 records in 2024-07-15T06:45:08.138 INFO:teuthology.orchestra.run.smithi134.stderr:10+0 records out 2024-07-15T06:45:08.139 INFO:teuthology.orchestra.run.smithi134.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.013492 s, 777 MB/s 2024-07-15T06:45:08.139 INFO:teuthology.orchestra.run.smithi134.stderr: stderr: 2024-07-15T06:45:08.139 INFO:teuthology.orchestra.run.smithi134.stderr:--> Zapping successful for: 2024-07-15T06:45:08.307 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:45:08 smithi134 bash[24068]: cluster 2024-07-15T06:45:06.949517+0000 mgr.y (mgr.14150) 263 : cluster [DBG] pgmap v242: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 90 KiB/s, 0 objects/s recovering 2024-07-15T06:45:08.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:45:08 smithi089 bash[21793]: cluster 2024-07-15T06:45:06.949517+0000 mgr.y (mgr.14150) 263 : cluster [DBG] pgmap v242: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 90 KiB/s, 0 objects/s recovering 2024-07-15T06:45:08.371 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:45:08 smithi089 bash[31372]: cluster 2024-07-15T06:45:06.949517+0000 mgr.y (mgr.14150) 263 : cluster [DBG] pgmap v242: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 90 KiB/s, 0 objects/s recovering 2024-07-15T06:45:09.159 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph orch daemon add osd smithi134:vg_nvme/lv_2 2024-07-15T06:45:10.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:45:10 smithi134 bash[24068]: cluster 2024-07-15T06:45:08.950078+0000 mgr.y (mgr.14150) 264 : cluster [DBG] pgmap v243: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-07-15T06:45:10.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:45:10 smithi134 bash[24068]: cephadm 2024-07-15T06:45:09.369772+0000 mgr.y (mgr.14150) 265 : cephadm [INF] Detected new or changed devices on smithi134 2024-07-15T06:45:10.808 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:45:10 smithi134 bash[24068]: audit 2024-07-15T06:45:09.378204+0000 mon.a (mon.0) 519 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:45:10.808 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:45:10 smithi134 bash[24068]: audit 2024-07-15T06:45:09.386591+0000 mon.a (mon.0) 520 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:45:10.808 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:45:10 smithi134 bash[24068]: audit 2024-07-15T06:45:09.388864+0000 mon.a (mon.0) 521 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:45:10.808 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:45:10 smithi134 bash[24068]: audit 2024-07-15T06:45:09.390719+0000 mon.a (mon.0) 522 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:45:10.808 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:45:10 smithi134 bash[24068]: cephadm 2024-07-15T06:45:09.392177+0000 mgr.y (mgr.14150) 266 : cephadm [INF] Adjusting osd_memory_target on smithi134 to 8632M 2024-07-15T06:45:10.808 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:45:10 smithi134 bash[24068]: audit 2024-07-15T06:45:09.400184+0000 mon.a (mon.0) 523 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:45:10.808 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:45:10 smithi134 bash[24068]: audit 2024-07-15T06:45:09.402339+0000 mon.a (mon.0) 524 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:45:10.808 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:45:10 smithi134 bash[24068]: audit 2024-07-15T06:45:09.403958+0000 mon.a (mon.0) 525 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:45:10.808 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:45:10 smithi134 bash[24068]: audit 2024-07-15T06:45:09.413862+0000 mon.a (mon.0) 526 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:45:10.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:45:10 smithi089 bash[21793]: cluster 2024-07-15T06:45:08.950078+0000 mgr.y (mgr.14150) 264 : cluster [DBG] pgmap v243: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-07-15T06:45:10.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:45:10 smithi089 bash[21793]: cephadm 2024-07-15T06:45:09.369772+0000 mgr.y (mgr.14150) 265 : cephadm [INF] Detected new or changed devices on smithi134 2024-07-15T06:45:10.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:45:10 smithi089 bash[21793]: audit 2024-07-15T06:45:09.378204+0000 mon.a (mon.0) 519 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:45:10.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:45:10 smithi089 bash[21793]: audit 2024-07-15T06:45:09.386591+0000 mon.a (mon.0) 520 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:45:10.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:45:10 smithi089 bash[21793]: audit 2024-07-15T06:45:09.388864+0000 mon.a (mon.0) 521 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:45:10.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:45:10 smithi089 bash[21793]: audit 2024-07-15T06:45:09.390719+0000 mon.a (mon.0) 522 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:45:10.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:45:10 smithi089 bash[21793]: cephadm 2024-07-15T06:45:09.392177+0000 mgr.y (mgr.14150) 266 : cephadm [INF] Adjusting osd_memory_target on smithi134 to 8632M 2024-07-15T06:45:10.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:45:10 smithi089 bash[21793]: audit 2024-07-15T06:45:09.400184+0000 mon.a (mon.0) 523 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:45:10.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:45:10 smithi089 bash[21793]: audit 2024-07-15T06:45:09.402339+0000 mon.a (mon.0) 524 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:45:10.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:45:10 smithi089 bash[21793]: audit 2024-07-15T06:45:09.403958+0000 mon.a (mon.0) 525 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:45:10.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:45:10 smithi089 bash[21793]: audit 2024-07-15T06:45:09.413862+0000 mon.a (mon.0) 526 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:45:10.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:45:10 smithi089 bash[31372]: cluster 2024-07-15T06:45:08.950078+0000 mgr.y (mgr.14150) 264 : cluster [DBG] pgmap v243: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-07-15T06:45:10.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:45:10 smithi089 bash[31372]: cephadm 2024-07-15T06:45:09.369772+0000 mgr.y (mgr.14150) 265 : cephadm [INF] Detected new or changed devices on smithi134 2024-07-15T06:45:10.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:45:10 smithi089 bash[31372]: audit 2024-07-15T06:45:09.378204+0000 mon.a (mon.0) 519 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:45:10.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:45:10 smithi089 bash[31372]: audit 2024-07-15T06:45:09.386591+0000 mon.a (mon.0) 520 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:45:10.873 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:45:10 smithi089 bash[31372]: audit 2024-07-15T06:45:09.388864+0000 mon.a (mon.0) 521 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:45:10.873 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:45:10 smithi089 bash[31372]: audit 2024-07-15T06:45:09.390719+0000 mon.a (mon.0) 522 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:45:10.873 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:45:10 smithi089 bash[31372]: cephadm 2024-07-15T06:45:09.392177+0000 mgr.y (mgr.14150) 266 : cephadm [INF] Adjusting osd_memory_target on smithi134 to 8632M 2024-07-15T06:45:10.873 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:45:10 smithi089 bash[31372]: audit 2024-07-15T06:45:09.400184+0000 mon.a (mon.0) 523 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:45:10.873 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:45:10 smithi089 bash[31372]: audit 2024-07-15T06:45:09.402339+0000 mon.a (mon.0) 524 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:45:10.873 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:45:10 smithi089 bash[31372]: audit 2024-07-15T06:45:09.403958+0000 mon.a (mon.0) 525 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:45:10.873 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:45:10 smithi089 bash[31372]: audit 2024-07-15T06:45:09.413862+0000 mon.a (mon.0) 526 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:45:12.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:45:12 smithi134 bash[24068]: cluster 2024-07-15T06:45:10.950619+0000 mgr.y (mgr.14150) 267 : cluster [DBG] pgmap v244: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 60 KiB/s, 0 objects/s recovering 2024-07-15T06:45:12.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:45:12 smithi089 bash[21793]: cluster 2024-07-15T06:45:10.950619+0000 mgr.y (mgr.14150) 267 : cluster [DBG] pgmap v244: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 60 KiB/s, 0 objects/s recovering 2024-07-15T06:45:12.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:45:12 smithi089 bash[31372]: cluster 2024-07-15T06:45:10.950619+0000 mgr.y (mgr.14150) 267 : cluster [DBG] pgmap v244: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 60 KiB/s, 0 objects/s recovering 2024-07-15T06:45:13.906 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.b/config 2024-07-15T06:45:14.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:45:14 smithi134 bash[24068]: cluster 2024-07-15T06:45:12.951137+0000 mgr.y (mgr.14150) 268 : cluster [DBG] pgmap v245: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 54 KiB/s, 0 objects/s recovering 2024-07-15T06:45:14.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:45:14 smithi089 bash[21793]: cluster 2024-07-15T06:45:12.951137+0000 mgr.y (mgr.14150) 268 : cluster [DBG] pgmap v245: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 54 KiB/s, 0 objects/s recovering 2024-07-15T06:45:14.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:45:14 smithi089 bash[31372]: cluster 2024-07-15T06:45:12.951137+0000 mgr.y (mgr.14150) 268 : cluster [DBG] pgmap v245: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 54 KiB/s, 0 objects/s recovering 2024-07-15T06:45:16.639 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:45:16 smithi134 bash[24068]: cluster 2024-07-15T06:45:14.951680+0000 mgr.y (mgr.14150) 269 : cluster [DBG] pgmap v246: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-07-15T06:45:16.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:45:16 smithi089 bash[21793]: cluster 2024-07-15T06:45:14.951680+0000 mgr.y (mgr.14150) 269 : cluster [DBG] pgmap v246: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-07-15T06:45:16.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:45:16 smithi089 bash[31372]: cluster 2024-07-15T06:45:14.951680+0000 mgr.y (mgr.14150) 269 : cluster [DBG] pgmap v246: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-07-15T06:45:17.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:45:17 smithi134 bash[24068]: audit 2024-07-15T06:45:16.966304+0000 mon.a (mon.0) 527 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-07-15T06:45:17.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:45:17 smithi134 bash[24068]: audit 2024-07-15T06:45:16.972274+0000 mon.a (mon.0) 528 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-07-15T06:45:17.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:45:17 smithi134 bash[24068]: audit 2024-07-15T06:45:16.974053+0000 mon.a (mon.0) 529 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:45:17.870 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:45:17 smithi089 bash[31372]: audit 2024-07-15T06:45:16.966304+0000 mon.a (mon.0) 527 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-07-15T06:45:17.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:45:17 smithi089 bash[31372]: audit 2024-07-15T06:45:16.972274+0000 mon.a (mon.0) 528 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-07-15T06:45:17.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:45:17 smithi089 bash[31372]: audit 2024-07-15T06:45:16.974053+0000 mon.a (mon.0) 529 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:45:17.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:45:17 smithi089 bash[21793]: audit 2024-07-15T06:45:16.966304+0000 mon.a (mon.0) 527 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-07-15T06:45:17.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:45:17 smithi089 bash[21793]: audit 2024-07-15T06:45:16.972274+0000 mon.a (mon.0) 528 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-07-15T06:45:17.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:45:17 smithi089 bash[21793]: audit 2024-07-15T06:45:16.974053+0000 mon.a (mon.0) 529 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:45:18.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:45:18 smithi134 bash[24068]: cluster 2024-07-15T06:45:16.952193+0000 mgr.y (mgr.14150) 270 : cluster [DBG] pgmap v247: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-15T06:45:18.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:45:18 smithi134 bash[24068]: audit 2024-07-15T06:45:16.960118+0000 mgr.y (mgr.14150) 271 : audit [DBG] from='client.24232 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi134:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T06:45:18.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:45:18 smithi089 bash[31372]: cluster 2024-07-15T06:45:16.952193+0000 mgr.y (mgr.14150) 270 : cluster [DBG] pgmap v247: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-15T06:45:18.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:45:18 smithi089 bash[31372]: audit 2024-07-15T06:45:16.960118+0000 mgr.y (mgr.14150) 271 : audit [DBG] from='client.24232 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi134:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T06:45:18.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:45:18 smithi089 bash[21793]: cluster 2024-07-15T06:45:16.952193+0000 mgr.y (mgr.14150) 270 : cluster [DBG] pgmap v247: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-15T06:45:18.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:45:18 smithi089 bash[21793]: audit 2024-07-15T06:45:16.960118+0000 mgr.y (mgr.14150) 271 : audit [DBG] from='client.24232 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi134:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T06:45:20.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:45:20 smithi134 bash[24068]: cluster 2024-07-15T06:45:18.952716+0000 mgr.y (mgr.14150) 272 : cluster [DBG] pgmap v248: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-15T06:45:20.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:45:20 smithi089 bash[21793]: cluster 2024-07-15T06:45:18.952716+0000 mgr.y (mgr.14150) 272 : cluster [DBG] pgmap v248: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-15T06:45:20.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:45:20 smithi089 bash[31372]: cluster 2024-07-15T06:45:18.952716+0000 mgr.y (mgr.14150) 272 : cluster [DBG] pgmap v248: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-15T06:45:22.766 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:45:22 smithi134 bash[24068]: cluster 2024-07-15T06:45:20.953237+0000 mgr.y (mgr.14150) 273 : cluster [DBG] pgmap v249: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-15T06:45:22.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:45:22 smithi089 bash[21793]: cluster 2024-07-15T06:45:20.953237+0000 mgr.y (mgr.14150) 273 : cluster [DBG] pgmap v249: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-15T06:45:22.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:45:22 smithi089 bash[31372]: cluster 2024-07-15T06:45:20.953237+0000 mgr.y (mgr.14150) 273 : cluster [DBG] pgmap v249: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-15T06:45:24.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:45:24 smithi134 bash[24068]: cluster 2024-07-15T06:45:22.953788+0000 mgr.y (mgr.14150) 274 : cluster [DBG] pgmap v250: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-15T06:45:24.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:45:24 smithi089 bash[21793]: cluster 2024-07-15T06:45:22.953788+0000 mgr.y (mgr.14150) 274 : cluster [DBG] pgmap v250: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-15T06:45:24.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:45:24 smithi089 bash[31372]: cluster 2024-07-15T06:45:22.953788+0000 mgr.y (mgr.14150) 274 : cluster [DBG] pgmap v250: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-15T06:45:25.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:45:25 smithi134 bash[24068]: audit 2024-07-15T06:45:25.286705+0000 mon.b (mon.1) 17 : audit [INF] from='client.? 172.21.15.134:0/1953452793' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "446cb8fa-8ddb-4f7a-a8a4-c7281e950f74"}]: dispatch 2024-07-15T06:45:25.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:45:25 smithi134 bash[24068]: audit 2024-07-15T06:45:25.289104+0000 mon.a (mon.0) 530 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "446cb8fa-8ddb-4f7a-a8a4-c7281e950f74"}]: dispatch 2024-07-15T06:45:25.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:45:25 smithi134 bash[24068]: audit 2024-07-15T06:45:25.302742+0000 mon.a (mon.0) 531 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "446cb8fa-8ddb-4f7a-a8a4-c7281e950f74"}]': finished 2024-07-15T06:45:25.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:45:25 smithi134 bash[24068]: cluster 2024-07-15T06:45:25.302853+0000 mon.a (mon.0) 532 : cluster [DBG] osdmap e43: 7 total, 6 up, 7 in 2024-07-15T06:45:25.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:45:25 smithi134 bash[24068]: audit 2024-07-15T06:45:25.303177+0000 mon.a (mon.0) 533 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-07-15T06:45:25.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:45:25 smithi089 bash[21793]: audit 2024-07-15T06:45:25.286705+0000 mon.b (mon.1) 17 : audit [INF] from='client.? 172.21.15.134:0/1953452793' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "446cb8fa-8ddb-4f7a-a8a4-c7281e950f74"}]: dispatch 2024-07-15T06:45:25.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:45:25 smithi089 bash[21793]: audit 2024-07-15T06:45:25.289104+0000 mon.a (mon.0) 530 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "446cb8fa-8ddb-4f7a-a8a4-c7281e950f74"}]: dispatch 2024-07-15T06:45:25.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:45:25 smithi089 bash[21793]: audit 2024-07-15T06:45:25.302742+0000 mon.a (mon.0) 531 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "446cb8fa-8ddb-4f7a-a8a4-c7281e950f74"}]': finished 2024-07-15T06:45:25.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:45:25 smithi089 bash[21793]: cluster 2024-07-15T06:45:25.302853+0000 mon.a (mon.0) 532 : cluster [DBG] osdmap e43: 7 total, 6 up, 7 in 2024-07-15T06:45:25.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:45:25 smithi089 bash[21793]: audit 2024-07-15T06:45:25.303177+0000 mon.a (mon.0) 533 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-07-15T06:45:25.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:45:25 smithi089 bash[31372]: audit 2024-07-15T06:45:25.286705+0000 mon.b (mon.1) 17 : audit [INF] from='client.? 172.21.15.134:0/1953452793' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "446cb8fa-8ddb-4f7a-a8a4-c7281e950f74"}]: dispatch 2024-07-15T06:45:25.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:45:25 smithi089 bash[31372]: audit 2024-07-15T06:45:25.289104+0000 mon.a (mon.0) 530 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "446cb8fa-8ddb-4f7a-a8a4-c7281e950f74"}]: dispatch 2024-07-15T06:45:25.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:45:25 smithi089 bash[31372]: audit 2024-07-15T06:45:25.302742+0000 mon.a (mon.0) 531 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "446cb8fa-8ddb-4f7a-a8a4-c7281e950f74"}]': finished 2024-07-15T06:45:25.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:45:25 smithi089 bash[31372]: cluster 2024-07-15T06:45:25.302853+0000 mon.a (mon.0) 532 : cluster [DBG] osdmap e43: 7 total, 6 up, 7 in 2024-07-15T06:45:25.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:45:25 smithi089 bash[31372]: audit 2024-07-15T06:45:25.303177+0000 mon.a (mon.0) 533 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-07-15T06:45:26.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:45:26 smithi134 bash[24068]: cluster 2024-07-15T06:45:24.954342+0000 mgr.y (mgr.14150) 275 : cluster [DBG] pgmap v251: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-15T06:45:26.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:45:26 smithi134 bash[24068]: audit 2024-07-15T06:45:25.912536+0000 mon.b (mon.1) 18 : audit [DBG] from='client.? 172.21.15.134:0/2294250185' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-07-15T06:45:26.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:45:26 smithi089 bash[21793]: cluster 2024-07-15T06:45:24.954342+0000 mgr.y (mgr.14150) 275 : cluster [DBG] pgmap v251: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-15T06:45:26.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:45:26 smithi089 bash[21793]: audit 2024-07-15T06:45:25.912536+0000 mon.b (mon.1) 18 : audit [DBG] from='client.? 172.21.15.134:0/2294250185' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-07-15T06:45:26.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:45:26 smithi089 bash[31372]: cluster 2024-07-15T06:45:24.954342+0000 mgr.y (mgr.14150) 275 : cluster [DBG] pgmap v251: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-15T06:45:26.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:45:26 smithi089 bash[31372]: audit 2024-07-15T06:45:25.912536+0000 mon.b (mon.1) 18 : audit [DBG] from='client.? 172.21.15.134:0/2294250185' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-07-15T06:45:28.582 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:45:28 smithi134 bash[24068]: cluster 2024-07-15T06:45:26.954845+0000 mgr.y (mgr.14150) 276 : cluster [DBG] pgmap v253: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-15T06:45:28.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:45:28 smithi089 bash[21793]: cluster 2024-07-15T06:45:26.954845+0000 mgr.y (mgr.14150) 276 : cluster [DBG] pgmap v253: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-15T06:45:28.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:45:28 smithi089 bash[31372]: cluster 2024-07-15T06:45:26.954845+0000 mgr.y (mgr.14150) 276 : cluster [DBG] pgmap v253: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-15T06:45:30.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:45:30 smithi134 bash[24068]: cluster 2024-07-15T06:45:28.955392+0000 mgr.y (mgr.14150) 277 : cluster [DBG] pgmap v254: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-15T06:45:30.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:45:30 smithi089 bash[21793]: cluster 2024-07-15T06:45:28.955392+0000 mgr.y (mgr.14150) 277 : cluster [DBG] pgmap v254: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-15T06:45:30.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:45:30 smithi089 bash[31372]: cluster 2024-07-15T06:45:28.955392+0000 mgr.y (mgr.14150) 277 : cluster [DBG] pgmap v254: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-15T06:45:32.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:45:32 smithi134 bash[24068]: cluster 2024-07-15T06:45:30.955934+0000 mgr.y (mgr.14150) 278 : cluster [DBG] pgmap v255: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-15T06:45:32.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:45:32 smithi089 bash[21793]: cluster 2024-07-15T06:45:30.955934+0000 mgr.y (mgr.14150) 278 : cluster [DBG] pgmap v255: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-15T06:45:32.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:45:32 smithi089 bash[31372]: cluster 2024-07-15T06:45:30.955934+0000 mgr.y (mgr.14150) 278 : cluster [DBG] pgmap v255: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-15T06:45:34.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:45:34 smithi134 bash[24068]: cluster 2024-07-15T06:45:32.956441+0000 mgr.y (mgr.14150) 279 : cluster [DBG] pgmap v256: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-15T06:45:34.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:45:34 smithi089 bash[21793]: cluster 2024-07-15T06:45:32.956441+0000 mgr.y (mgr.14150) 279 : cluster [DBG] pgmap v256: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-15T06:45:34.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:45:34 smithi089 bash[31372]: cluster 2024-07-15T06:45:32.956441+0000 mgr.y (mgr.14150) 279 : cluster [DBG] pgmap v256: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-15T06:45:36.719 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:45:36 smithi134 bash[24068]: cluster 2024-07-15T06:45:34.956919+0000 mgr.y (mgr.14150) 280 : cluster [DBG] pgmap v257: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-15T06:45:36.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:45:36 smithi089 bash[21793]: cluster 2024-07-15T06:45:34.956919+0000 mgr.y (mgr.14150) 280 : cluster [DBG] pgmap v257: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-15T06:45:36.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:45:36 smithi089 bash[31372]: cluster 2024-07-15T06:45:34.956919+0000 mgr.y (mgr.14150) 280 : cluster [DBG] pgmap v257: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-15T06:45:38.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:45:38 smithi134 bash[24068]: cluster 2024-07-15T06:45:36.957432+0000 mgr.y (mgr.14150) 281 : cluster [DBG] pgmap v258: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-15T06:45:38.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:45:38 smithi134 bash[24068]: audit 2024-07-15T06:45:37.697345+0000 mon.a (mon.0) 534 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-07-15T06:45:38.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:45:38 smithi134 bash[24068]: audit 2024-07-15T06:45:37.698959+0000 mon.a (mon.0) 535 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:45:38.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:45:38 smithi089 bash[21793]: cluster 2024-07-15T06:45:36.957432+0000 mgr.y (mgr.14150) 281 : cluster [DBG] pgmap v258: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-15T06:45:38.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:45:38 smithi089 bash[21793]: audit 2024-07-15T06:45:37.697345+0000 mon.a (mon.0) 534 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-07-15T06:45:38.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:45:38 smithi089 bash[21793]: audit 2024-07-15T06:45:37.698959+0000 mon.a (mon.0) 535 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:45:38.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:45:38 smithi089 bash[31372]: cluster 2024-07-15T06:45:36.957432+0000 mgr.y (mgr.14150) 281 : cluster [DBG] pgmap v258: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-15T06:45:38.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:45:38 smithi089 bash[31372]: audit 2024-07-15T06:45:37.697345+0000 mon.a (mon.0) 534 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-07-15T06:45:38.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:45:38 smithi089 bash[31372]: audit 2024-07-15T06:45:37.698959+0000 mon.a (mon.0) 535 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:45:39.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:45:39 smithi134 bash[24068]: cephadm 2024-07-15T06:45:37.700423+0000 mgr.y (mgr.14150) 282 : cephadm [INF] Deploying daemon osd.6 on smithi134 2024-07-15T06:45:39.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:45:39 smithi089 bash[21793]: cephadm 2024-07-15T06:45:37.700423+0000 mgr.y (mgr.14150) 282 : cephadm [INF] Deploying daemon osd.6 on smithi134 2024-07-15T06:45:39.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:45:39 smithi089 bash[31372]: cephadm 2024-07-15T06:45:37.700423+0000 mgr.y (mgr.14150) 282 : cephadm [INF] Deploying daemon osd.6 on smithi134 2024-07-15T06:45:40.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:45:40 smithi134 bash[24068]: cluster 2024-07-15T06:45:38.957960+0000 mgr.y (mgr.14150) 283 : cluster [DBG] pgmap v259: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-15T06:45:40.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:45:40 smithi089 bash[21793]: cluster 2024-07-15T06:45:38.957960+0000 mgr.y (mgr.14150) 283 : cluster [DBG] pgmap v259: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-15T06:45:40.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:45:40 smithi089 bash[31372]: cluster 2024-07-15T06:45:38.957960+0000 mgr.y (mgr.14150) 283 : cluster [DBG] pgmap v259: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-15T06:45:42.057 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:45:41 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:45:42.057 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:45:41 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:45:42.057 INFO:journalctl@ceph.osd.4.smithi134.stdout:Jul 15 06:45:41 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:45:42.058 INFO:journalctl@ceph.osd.5.smithi134.stdout:Jul 15 06:45:41 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:45:42.462 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:45:42 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:45:42.462 INFO:journalctl@ceph.osd.4.smithi134.stdout:Jul 15 06:45:42 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:45:42.462 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:45:42 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:45:42.463 INFO:journalctl@ceph.osd.5.smithi134.stdout:Jul 15 06:45:42 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:45:42.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:45:42 smithi134 bash[24068]: cluster 2024-07-15T06:45:40.958500+0000 mgr.y (mgr.14150) 284 : cluster [DBG] pgmap v260: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-15T06:45:42.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:45:42 smithi134 bash[24068]: audit 2024-07-15T06:45:42.293439+0000 mon.a (mon.0) 536 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:45:42.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:45:42 smithi134 bash[24068]: audit 2024-07-15T06:45:42.300714+0000 mon.a (mon.0) 537 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:45:42.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:45:42 smithi134 bash[24068]: audit 2024-07-15T06:45:42.308643+0000 mon.a (mon.0) 538 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:45:42.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:45:42 smithi089 bash[21793]: cluster 2024-07-15T06:45:40.958500+0000 mgr.y (mgr.14150) 284 : cluster [DBG] pgmap v260: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-15T06:45:42.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:45:42 smithi089 bash[21793]: audit 2024-07-15T06:45:42.293439+0000 mon.a (mon.0) 536 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:45:42.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:45:42 smithi089 bash[21793]: audit 2024-07-15T06:45:42.300714+0000 mon.a (mon.0) 537 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:45:42.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:45:42 smithi089 bash[21793]: audit 2024-07-15T06:45:42.308643+0000 mon.a (mon.0) 538 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:45:42.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:45:42 smithi089 bash[31372]: cluster 2024-07-15T06:45:40.958500+0000 mgr.y (mgr.14150) 284 : cluster [DBG] pgmap v260: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-15T06:45:42.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:45:42 smithi089 bash[31372]: audit 2024-07-15T06:45:42.293439+0000 mon.a (mon.0) 536 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:45:42.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:45:42 smithi089 bash[31372]: audit 2024-07-15T06:45:42.300714+0000 mon.a (mon.0) 537 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:45:42.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:45:42 smithi089 bash[31372]: audit 2024-07-15T06:45:42.308643+0000 mon.a (mon.0) 538 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:45:44.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:45:44 smithi134 bash[24068]: cluster 2024-07-15T06:45:42.958986+0000 mgr.y (mgr.14150) 285 : cluster [DBG] pgmap v261: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-15T06:45:44.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:45:44 smithi089 bash[21793]: cluster 2024-07-15T06:45:42.958986+0000 mgr.y (mgr.14150) 285 : cluster [DBG] pgmap v261: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-15T06:45:44.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:45:44 smithi089 bash[31372]: cluster 2024-07-15T06:45:42.958986+0000 mgr.y (mgr.14150) 285 : cluster [DBG] pgmap v261: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-15T06:45:46.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:45:46 smithi134 bash[24068]: cluster 2024-07-15T06:45:44.959528+0000 mgr.y (mgr.14150) 286 : cluster [DBG] pgmap v262: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-15T06:45:46.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:45:46 smithi089 bash[21793]: cluster 2024-07-15T06:45:44.959528+0000 mgr.y (mgr.14150) 286 : cluster [DBG] pgmap v262: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-15T06:45:46.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:45:46 smithi089 bash[31372]: cluster 2024-07-15T06:45:44.959528+0000 mgr.y (mgr.14150) 286 : cluster [DBG] pgmap v262: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-15T06:45:48.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:45:48 smithi134 bash[24068]: cluster 2024-07-15T06:45:46.960008+0000 mgr.y (mgr.14150) 287 : cluster [DBG] pgmap v263: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-15T06:45:48.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:45:48 smithi134 bash[24068]: audit 2024-07-15T06:45:48.365518+0000 mon.a (mon.0) 539 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:45:48.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:45:48 smithi134 bash[24068]: audit 2024-07-15T06:45:48.374084+0000 mon.a (mon.0) 540 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:45:48.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:45:48 smithi089 bash[21793]: cluster 2024-07-15T06:45:46.960008+0000 mgr.y (mgr.14150) 287 : cluster [DBG] pgmap v263: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-15T06:45:48.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:45:48 smithi089 bash[21793]: audit 2024-07-15T06:45:48.365518+0000 mon.a (mon.0) 539 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:45:48.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:45:48 smithi089 bash[21793]: audit 2024-07-15T06:45:48.374084+0000 mon.a (mon.0) 540 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:45:48.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:45:48 smithi089 bash[31372]: cluster 2024-07-15T06:45:46.960008+0000 mgr.y (mgr.14150) 287 : cluster [DBG] pgmap v263: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-15T06:45:48.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:45:48 smithi089 bash[31372]: audit 2024-07-15T06:45:48.365518+0000 mon.a (mon.0) 539 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:45:48.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:45:48 smithi089 bash[31372]: audit 2024-07-15T06:45:48.374084+0000 mon.a (mon.0) 540 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:45:49.557 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:45:49 smithi134 bash[24068]: audit 2024-07-15T06:45:48.795496+0000 mon.a (mon.0) 541 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:45:49.557 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:45:49 smithi134 bash[24068]: audit 2024-07-15T06:45:48.796918+0000 mon.a (mon.0) 542 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:45:49.557 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:45:49 smithi134 bash[24068]: audit 2024-07-15T06:45:48.807584+0000 mon.a (mon.0) 543 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:45:49.557 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:45:49 smithi134 bash[24068]: audit 2024-07-15T06:45:49.041422+0000 mon.b (mon.1) 19 : audit [INF] from='osd.6 [v2:172.21.15.134:6816/4010812142,v1:172.21.15.134:6817/4010812142]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-07-15T06:45:49.557 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:45:49 smithi134 bash[24068]: audit 2024-07-15T06:45:49.043591+0000 mon.a (mon.0) 544 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-07-15T06:45:49.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:45:49 smithi089 bash[21793]: audit 2024-07-15T06:45:48.795496+0000 mon.a (mon.0) 541 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:45:49.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:45:49 smithi089 bash[21793]: audit 2024-07-15T06:45:48.796918+0000 mon.a (mon.0) 542 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:45:49.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:45:49 smithi089 bash[21793]: audit 2024-07-15T06:45:48.807584+0000 mon.a (mon.0) 543 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:45:49.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:45:49 smithi089 bash[21793]: audit 2024-07-15T06:45:49.041422+0000 mon.b (mon.1) 19 : audit [INF] from='osd.6 [v2:172.21.15.134:6816/4010812142,v1:172.21.15.134:6817/4010812142]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-07-15T06:45:49.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:45:49 smithi089 bash[21793]: audit 2024-07-15T06:45:49.043591+0000 mon.a (mon.0) 544 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-07-15T06:45:49.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:45:49 smithi089 bash[31372]: audit 2024-07-15T06:45:48.795496+0000 mon.a (mon.0) 541 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:45:49.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:45:49 smithi089 bash[31372]: audit 2024-07-15T06:45:48.796918+0000 mon.a (mon.0) 542 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:45:49.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:45:49 smithi089 bash[31372]: audit 2024-07-15T06:45:48.807584+0000 mon.a (mon.0) 543 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:45:49.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:45:49 smithi089 bash[31372]: audit 2024-07-15T06:45:49.041422+0000 mon.b (mon.1) 19 : audit [INF] from='osd.6 [v2:172.21.15.134:6816/4010812142,v1:172.21.15.134:6817/4010812142]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-07-15T06:45:49.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:45:49 smithi089 bash[31372]: audit 2024-07-15T06:45:49.043591+0000 mon.a (mon.0) 544 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-07-15T06:45:50.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:45:50 smithi134 bash[24068]: cluster 2024-07-15T06:45:48.960553+0000 mgr.y (mgr.14150) 288 : cluster [DBG] pgmap v264: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-15T06:45:50.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:45:50 smithi134 bash[24068]: audit 2024-07-15T06:45:49.821027+0000 mon.b (mon.1) 20 : audit [INF] from='osd.6 [v2:172.21.15.134:6816/4010812142,v1:172.21.15.134:6817/4010812142]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]: dispatch 2024-07-15T06:45:50.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:45:50 smithi134 bash[24068]: audit 2024-07-15T06:45:49.821077+0000 mon.a (mon.0) 545 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-07-15T06:45:50.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:45:50 smithi134 bash[24068]: cluster 2024-07-15T06:45:49.821242+0000 mon.a (mon.0) 546 : cluster [DBG] osdmap e44: 7 total, 6 up, 7 in 2024-07-15T06:45:50.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:45:50 smithi134 bash[24068]: audit 2024-07-15T06:45:49.821543+0000 mon.a (mon.0) 547 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-07-15T06:45:50.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:45:50 smithi134 bash[24068]: audit 2024-07-15T06:45:49.823430+0000 mon.a (mon.0) 548 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]: dispatch 2024-07-15T06:45:50.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:45:50 smithi089 bash[21793]: cluster 2024-07-15T06:45:48.960553+0000 mgr.y (mgr.14150) 288 : cluster [DBG] pgmap v264: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-15T06:45:50.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:45:50 smithi089 bash[21793]: audit 2024-07-15T06:45:49.821027+0000 mon.b (mon.1) 20 : audit [INF] from='osd.6 [v2:172.21.15.134:6816/4010812142,v1:172.21.15.134:6817/4010812142]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]: dispatch 2024-07-15T06:45:50.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:45:50 smithi089 bash[21793]: audit 2024-07-15T06:45:49.821077+0000 mon.a (mon.0) 545 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-07-15T06:45:50.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:45:50 smithi089 bash[21793]: cluster 2024-07-15T06:45:49.821242+0000 mon.a (mon.0) 546 : cluster [DBG] osdmap e44: 7 total, 6 up, 7 in 2024-07-15T06:45:50.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:45:50 smithi089 bash[21793]: audit 2024-07-15T06:45:49.821543+0000 mon.a (mon.0) 547 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-07-15T06:45:50.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:45:50 smithi089 bash[21793]: audit 2024-07-15T06:45:49.823430+0000 mon.a (mon.0) 548 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]: dispatch 2024-07-15T06:45:50.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:45:50 smithi089 bash[31372]: cluster 2024-07-15T06:45:48.960553+0000 mgr.y (mgr.14150) 288 : cluster [DBG] pgmap v264: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-15T06:45:50.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:45:50 smithi089 bash[31372]: audit 2024-07-15T06:45:49.821027+0000 mon.b (mon.1) 20 : audit [INF] from='osd.6 [v2:172.21.15.134:6816/4010812142,v1:172.21.15.134:6817/4010812142]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]: dispatch 2024-07-15T06:45:50.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:45:50 smithi089 bash[31372]: audit 2024-07-15T06:45:49.821077+0000 mon.a (mon.0) 545 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-07-15T06:45:50.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:45:50 smithi089 bash[31372]: cluster 2024-07-15T06:45:49.821242+0000 mon.a (mon.0) 546 : cluster [DBG] osdmap e44: 7 total, 6 up, 7 in 2024-07-15T06:45:50.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:45:50 smithi089 bash[31372]: audit 2024-07-15T06:45:49.821543+0000 mon.a (mon.0) 547 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-07-15T06:45:50.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:45:50 smithi089 bash[31372]: audit 2024-07-15T06:45:49.823430+0000 mon.a (mon.0) 548 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]: dispatch 2024-07-15T06:45:52.057 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:45:51 smithi134 bash[24068]: audit 2024-07-15T06:45:50.822478+0000 mon.a (mon.0) 549 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]': finished 2024-07-15T06:45:52.057 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:45:51 smithi134 bash[24068]: cluster 2024-07-15T06:45:50.822692+0000 mon.a (mon.0) 550 : cluster [DBG] osdmap e45: 7 total, 6 up, 7 in 2024-07-15T06:45:52.057 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:45:51 smithi134 bash[24068]: audit 2024-07-15T06:45:50.822944+0000 mon.a (mon.0) 551 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-07-15T06:45:52.057 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:45:51 smithi134 bash[24068]: audit 2024-07-15T06:45:50.830082+0000 mon.a (mon.0) 552 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-07-15T06:45:52.057 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:45:51 smithi134 bash[24068]: cluster 2024-07-15T06:45:50.961119+0000 mgr.y (mgr.14150) 289 : cluster [DBG] pgmap v267: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-15T06:45:52.057 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:45:51 smithi134 bash[24068]: cluster 2024-07-15T06:45:51.325888+0000 mon.a (mon.0) 553 : cluster [INF] osd.6 [v2:172.21.15.134:6816/4010812142,v1:172.21.15.134:6817/4010812142] boot 2024-07-15T06:45:52.057 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:45:51 smithi134 bash[24068]: cluster 2024-07-15T06:45:51.326070+0000 mon.a (mon.0) 554 : cluster [DBG] osdmap e46: 7 total, 7 up, 7 in 2024-07-15T06:45:52.057 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:45:51 smithi134 bash[24068]: audit 2024-07-15T06:45:51.326603+0000 mon.a (mon.0) 555 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-07-15T06:45:52.112 INFO:teuthology.orchestra.run.smithi134.stdout:Created osd(s) 6 on host 'smithi134' 2024-07-15T06:45:52.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:45:51 smithi089 bash[21793]: audit 2024-07-15T06:45:50.822478+0000 mon.a (mon.0) 549 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]': finished 2024-07-15T06:45:52.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:45:51 smithi089 bash[21793]: cluster 2024-07-15T06:45:50.822692+0000 mon.a (mon.0) 550 : cluster [DBG] osdmap e45: 7 total, 6 up, 7 in 2024-07-15T06:45:52.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:45:51 smithi089 bash[21793]: audit 2024-07-15T06:45:50.822944+0000 mon.a (mon.0) 551 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-07-15T06:45:52.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:45:51 smithi089 bash[21793]: audit 2024-07-15T06:45:50.830082+0000 mon.a (mon.0) 552 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-07-15T06:45:52.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:45:51 smithi089 bash[21793]: cluster 2024-07-15T06:45:50.961119+0000 mgr.y (mgr.14150) 289 : cluster [DBG] pgmap v267: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-15T06:45:52.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:45:51 smithi089 bash[21793]: cluster 2024-07-15T06:45:51.325888+0000 mon.a (mon.0) 553 : cluster [INF] osd.6 [v2:172.21.15.134:6816/4010812142,v1:172.21.15.134:6817/4010812142] boot 2024-07-15T06:45:52.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:45:51 smithi089 bash[21793]: cluster 2024-07-15T06:45:51.326070+0000 mon.a (mon.0) 554 : cluster [DBG] osdmap e46: 7 total, 7 up, 7 in 2024-07-15T06:45:52.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:45:51 smithi089 bash[21793]: audit 2024-07-15T06:45:51.326603+0000 mon.a (mon.0) 555 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-07-15T06:45:52.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:45:51 smithi089 bash[31372]: audit 2024-07-15T06:45:50.822478+0000 mon.a (mon.0) 549 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]': finished 2024-07-15T06:45:52.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:45:51 smithi089 bash[31372]: cluster 2024-07-15T06:45:50.822692+0000 mon.a (mon.0) 550 : cluster [DBG] osdmap e45: 7 total, 6 up, 7 in 2024-07-15T06:45:52.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:45:51 smithi089 bash[31372]: audit 2024-07-15T06:45:50.822944+0000 mon.a (mon.0) 551 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-07-15T06:45:52.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:45:51 smithi089 bash[31372]: audit 2024-07-15T06:45:50.830082+0000 mon.a (mon.0) 552 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-07-15T06:45:52.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:45:51 smithi089 bash[31372]: cluster 2024-07-15T06:45:50.961119+0000 mgr.y (mgr.14150) 289 : cluster [DBG] pgmap v267: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-15T06:45:52.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:45:51 smithi089 bash[31372]: cluster 2024-07-15T06:45:51.325888+0000 mon.a (mon.0) 553 : cluster [INF] osd.6 [v2:172.21.15.134:6816/4010812142,v1:172.21.15.134:6817/4010812142] boot 2024-07-15T06:45:52.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:45:51 smithi089 bash[31372]: cluster 2024-07-15T06:45:51.326070+0000 mon.a (mon.0) 554 : cluster [DBG] osdmap e46: 7 total, 7 up, 7 in 2024-07-15T06:45:52.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:45:51 smithi089 bash[31372]: audit 2024-07-15T06:45:51.326603+0000 mon.a (mon.0) 555 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-07-15T06:45:52.752 DEBUG:teuthology.orchestra.run.smithi134:osd.6> sudo journalctl -f -n 0 -u ceph-75ba02bc-4274-11ef-bcad-c7b262605968@osd.6.service 2024-07-15T06:45:52.755 INFO:tasks.cephadm:Deploying osd.7 on smithi134 with /dev/vg_nvme/lv_1... 2024-07-15T06:45:52.755 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-07-15T06:45:53.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:45:52 smithi134 bash[24068]: cluster 2024-07-15T06:45:50.039604+0000 osd.6 (osd.6) 1 : cluster [DBG] purged_snaps scrub starts 2024-07-15T06:45:53.057 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:45:52 smithi134 bash[24068]: cluster 2024-07-15T06:45:50.039717+0000 osd.6 (osd.6) 2 : cluster [DBG] purged_snaps scrub ok 2024-07-15T06:45:53.057 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:45:52 smithi134 bash[24068]: audit 2024-07-15T06:45:52.096206+0000 mon.a (mon.0) 556 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:45:53.057 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:45:52 smithi134 bash[24068]: audit 2024-07-15T06:45:52.103870+0000 mon.a (mon.0) 557 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:45:53.057 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:45:52 smithi134 bash[24068]: audit 2024-07-15T06:45:52.111963+0000 mon.a (mon.0) 558 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:45:53.057 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:45:52 smithi134 bash[24068]: cluster 2024-07-15T06:45:52.323301+0000 mon.a (mon.0) 559 : cluster [DBG] osdmap e47: 7 total, 7 up, 7 in 2024-07-15T06:45:53.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:45:52 smithi089 bash[21793]: cluster 2024-07-15T06:45:50.039604+0000 osd.6 (osd.6) 1 : cluster [DBG] purged_snaps scrub starts 2024-07-15T06:45:53.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:45:52 smithi089 bash[21793]: cluster 2024-07-15T06:45:50.039717+0000 osd.6 (osd.6) 2 : cluster [DBG] purged_snaps scrub ok 2024-07-15T06:45:53.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:45:52 smithi089 bash[21793]: audit 2024-07-15T06:45:52.096206+0000 mon.a (mon.0) 556 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:45:53.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:45:52 smithi089 bash[21793]: audit 2024-07-15T06:45:52.103870+0000 mon.a (mon.0) 557 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:45:53.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:45:52 smithi089 bash[21793]: audit 2024-07-15T06:45:52.111963+0000 mon.a (mon.0) 558 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:45:53.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:45:52 smithi089 bash[21793]: cluster 2024-07-15T06:45:52.323301+0000 mon.a (mon.0) 559 : cluster [DBG] osdmap e47: 7 total, 7 up, 7 in 2024-07-15T06:45:53.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:45:52 smithi089 bash[31372]: cluster 2024-07-15T06:45:50.039604+0000 osd.6 (osd.6) 1 : cluster [DBG] purged_snaps scrub starts 2024-07-15T06:45:53.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:45:52 smithi089 bash[31372]: cluster 2024-07-15T06:45:50.039717+0000 osd.6 (osd.6) 2 : cluster [DBG] purged_snaps scrub ok 2024-07-15T06:45:53.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:45:52 smithi089 bash[31372]: audit 2024-07-15T06:45:52.096206+0000 mon.a (mon.0) 556 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:45:53.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:45:52 smithi089 bash[31372]: audit 2024-07-15T06:45:52.103870+0000 mon.a (mon.0) 557 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:45:53.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:45:52 smithi089 bash[31372]: audit 2024-07-15T06:45:52.111963+0000 mon.a (mon.0) 558 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:45:53.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:45:52 smithi089 bash[31372]: cluster 2024-07-15T06:45:52.323301+0000 mon.a (mon.0) 559 : cluster [DBG] osdmap e47: 7 total, 7 up, 7 in 2024-07-15T06:45:54.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:45:54 smithi089 bash[21793]: cluster 2024-07-15T06:45:52.961721+0000 mgr.y (mgr.14150) 290 : cluster [DBG] pgmap v270: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-15T06:45:54.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:45:54 smithi089 bash[21793]: cluster 2024-07-15T06:45:53.327058+0000 mon.a (mon.0) 560 : cluster [DBG] osdmap e48: 7 total, 7 up, 7 in 2024-07-15T06:45:54.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:45:54 smithi089 bash[31372]: cluster 2024-07-15T06:45:52.961721+0000 mgr.y (mgr.14150) 290 : cluster [DBG] pgmap v270: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-15T06:45:54.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:45:54 smithi089 bash[31372]: cluster 2024-07-15T06:45:53.327058+0000 mon.a (mon.0) 560 : cluster [DBG] osdmap e48: 7 total, 7 up, 7 in 2024-07-15T06:45:54.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:45:54 smithi134 bash[24068]: cluster 2024-07-15T06:45:52.961721+0000 mgr.y (mgr.14150) 290 : cluster [DBG] pgmap v270: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-15T06:45:54.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:45:54 smithi134 bash[24068]: cluster 2024-07-15T06:45:53.327058+0000 mon.a (mon.0) 560 : cluster [DBG] osdmap e48: 7 total, 7 up, 7 in 2024-07-15T06:45:55.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:45:55 smithi089 bash[21793]: cluster 2024-07-15T06:45:54.327969+0000 mon.a (mon.0) 561 : cluster [DBG] osdmap e49: 7 total, 7 up, 7 in 2024-07-15T06:45:55.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:45:55 smithi089 bash[31372]: cluster 2024-07-15T06:45:54.327969+0000 mon.a (mon.0) 561 : cluster [DBG] osdmap e49: 7 total, 7 up, 7 in 2024-07-15T06:45:55.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:45:55 smithi134 bash[24068]: cluster 2024-07-15T06:45:54.327969+0000 mon.a (mon.0) 561 : cluster [DBG] osdmap e49: 7 total, 7 up, 7 in 2024-07-15T06:45:56.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:45:56 smithi089 bash[31372]: cluster 2024-07-15T06:45:54.962304+0000 mgr.y (mgr.14150) 291 : cluster [DBG] pgmap v273: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 144 KiB/s, 0 objects/s recovering 2024-07-15T06:45:56.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:45:56 smithi089 bash[31372]: cluster 2024-07-15T06:45:55.353818+0000 mon.a (mon.0) 562 : cluster [DBG] osdmap e50: 7 total, 7 up, 7 in 2024-07-15T06:45:56.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:45:56 smithi089 bash[21793]: cluster 2024-07-15T06:45:54.962304+0000 mgr.y (mgr.14150) 291 : cluster [DBG] pgmap v273: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 144 KiB/s, 0 objects/s recovering 2024-07-15T06:45:56.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:45:56 smithi089 bash[21793]: cluster 2024-07-15T06:45:55.353818+0000 mon.a (mon.0) 562 : cluster [DBG] osdmap e50: 7 total, 7 up, 7 in 2024-07-15T06:45:56.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:45:56 smithi134 bash[24068]: cluster 2024-07-15T06:45:54.962304+0000 mgr.y (mgr.14150) 291 : cluster [DBG] pgmap v273: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 144 KiB/s, 0 objects/s recovering 2024-07-15T06:45:56.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:45:56 smithi134 bash[24068]: cluster 2024-07-15T06:45:55.353818+0000 mon.a (mon.0) 562 : cluster [DBG] osdmap e50: 7 total, 7 up, 7 in 2024-07-15T06:45:56.879 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.b/config 2024-07-15T06:45:58.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:45:58 smithi089 bash[21793]: cluster 2024-07-15T06:45:56.962848+0000 mgr.y (mgr.14150) 292 : cluster [DBG] pgmap v275: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 124 KiB/s, 0 objects/s recovering 2024-07-15T06:45:58.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:45:58 smithi089 bash[31372]: cluster 2024-07-15T06:45:56.962848+0000 mgr.y (mgr.14150) 292 : cluster [DBG] pgmap v275: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 124 KiB/s, 0 objects/s recovering 2024-07-15T06:45:58.622 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:45:58 smithi134 bash[24068]: cluster 2024-07-15T06:45:56.962848+0000 mgr.y (mgr.14150) 292 : cluster [DBG] pgmap v275: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 124 KiB/s, 0 objects/s recovering 2024-07-15T06:46:00.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:00 smithi089 bash[21793]: cluster 2024-07-15T06:45:58.963443+0000 mgr.y (mgr.14150) 293 : cluster [DBG] pgmap v276: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-07-15T06:46:00.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:00 smithi089 bash[31372]: cluster 2024-07-15T06:45:58.963443+0000 mgr.y (mgr.14150) 293 : cluster [DBG] pgmap v276: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-07-15T06:46:00.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:00 smithi134 bash[24068]: cluster 2024-07-15T06:45:58.963443+0000 mgr.y (mgr.14150) 293 : cluster [DBG] pgmap v276: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-07-15T06:46:01.692 INFO:teuthology.orchestra.run.smithi134.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-07-15T06:46:01.897 INFO:teuthology.orchestra.run.smithi134.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-07-15T06:46:01.913 INFO:teuthology.orchestra.run.smithi134.stderr: stderr: 10+0 records in 2024-07-15T06:46:01.913 INFO:teuthology.orchestra.run.smithi134.stderr:10+0 records out 2024-07-15T06:46:01.913 INFO:teuthology.orchestra.run.smithi134.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0137849 s, 761 MB/s 2024-07-15T06:46:01.913 INFO:teuthology.orchestra.run.smithi134.stderr:--> Zapping successful for: 2024-07-15T06:46:02.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:02 smithi089 bash[21793]: cluster 2024-07-15T06:46:00.963957+0000 mgr.y (mgr.14150) 294 : cluster [DBG] pgmap v277: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 75 KiB/s, 0 objects/s recovering 2024-07-15T06:46:02.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:02 smithi089 bash[31372]: cluster 2024-07-15T06:46:00.963957+0000 mgr.y (mgr.14150) 294 : cluster [DBG] pgmap v277: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 75 KiB/s, 0 objects/s recovering 2024-07-15T06:46:02.684 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:02 smithi134 bash[24068]: cluster 2024-07-15T06:46:00.963957+0000 mgr.y (mgr.14150) 294 : cluster [DBG] pgmap v277: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 75 KiB/s, 0 objects/s recovering 2024-07-15T06:46:02.928 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph orch daemon add osd smithi134:vg_nvme/lv_1 2024-07-15T06:46:04.557 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:04 smithi134 bash[24068]: cluster 2024-07-15T06:46:02.964535+0000 mgr.y (mgr.14150) 295 : cluster [DBG] pgmap v278: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-15T06:46:04.557 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:04 smithi134 bash[24068]: cephadm 2024-07-15T06:46:03.190662+0000 mgr.y (mgr.14150) 296 : cephadm [INF] Detected new or changed devices on smithi134 2024-07-15T06:46:04.557 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:04 smithi134 bash[24068]: audit 2024-07-15T06:46:03.199156+0000 mon.a (mon.0) 563 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:46:04.557 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:04 smithi134 bash[24068]: audit 2024-07-15T06:46:03.207639+0000 mon.a (mon.0) 564 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:46:04.557 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:04 smithi134 bash[24068]: audit 2024-07-15T06:46:03.209817+0000 mon.a (mon.0) 565 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:46:04.557 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:04 smithi134 bash[24068]: audit 2024-07-15T06:46:03.211484+0000 mon.a (mon.0) 566 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:46:04.557 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:04 smithi134 bash[24068]: audit 2024-07-15T06:46:03.213168+0000 mon.a (mon.0) 567 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:46:04.558 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:04 smithi134 bash[24068]: cephadm 2024-07-15T06:46:03.214493+0000 mgr.y (mgr.14150) 297 : cephadm [INF] Adjusting osd_memory_target on smithi134 to 5755M 2024-07-15T06:46:04.558 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:04 smithi134 bash[24068]: audit 2024-07-15T06:46:03.222607+0000 mon.a (mon.0) 568 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:46:04.558 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:04 smithi134 bash[24068]: audit 2024-07-15T06:46:03.224714+0000 mon.a (mon.0) 569 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:46:04.558 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:04 smithi134 bash[24068]: audit 2024-07-15T06:46:03.226438+0000 mon.a (mon.0) 570 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:46:04.558 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:04 smithi134 bash[24068]: audit 2024-07-15T06:46:03.235907+0000 mon.a (mon.0) 571 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:46:04.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:04 smithi089 bash[21793]: cluster 2024-07-15T06:46:02.964535+0000 mgr.y (mgr.14150) 295 : cluster [DBG] pgmap v278: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-15T06:46:04.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:04 smithi089 bash[21793]: cephadm 2024-07-15T06:46:03.190662+0000 mgr.y (mgr.14150) 296 : cephadm [INF] Detected new or changed devices on smithi134 2024-07-15T06:46:04.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:04 smithi089 bash[21793]: audit 2024-07-15T06:46:03.199156+0000 mon.a (mon.0) 563 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:46:04.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:04 smithi089 bash[21793]: audit 2024-07-15T06:46:03.207639+0000 mon.a (mon.0) 564 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:46:04.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:04 smithi089 bash[21793]: audit 2024-07-15T06:46:03.209817+0000 mon.a (mon.0) 565 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:46:04.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:04 smithi089 bash[21793]: audit 2024-07-15T06:46:03.211484+0000 mon.a (mon.0) 566 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:46:04.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:04 smithi089 bash[21793]: audit 2024-07-15T06:46:03.213168+0000 mon.a (mon.0) 567 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:46:04.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:04 smithi089 bash[21793]: cephadm 2024-07-15T06:46:03.214493+0000 mgr.y (mgr.14150) 297 : cephadm [INF] Adjusting osd_memory_target on smithi134 to 5755M 2024-07-15T06:46:04.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:04 smithi089 bash[21793]: audit 2024-07-15T06:46:03.222607+0000 mon.a (mon.0) 568 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:46:04.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:04 smithi089 bash[21793]: audit 2024-07-15T06:46:03.224714+0000 mon.a (mon.0) 569 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:46:04.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:04 smithi089 bash[21793]: audit 2024-07-15T06:46:03.226438+0000 mon.a (mon.0) 570 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:46:04.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:04 smithi089 bash[21793]: audit 2024-07-15T06:46:03.235907+0000 mon.a (mon.0) 571 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:46:04.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:04 smithi089 bash[31372]: cluster 2024-07-15T06:46:02.964535+0000 mgr.y (mgr.14150) 295 : cluster [DBG] pgmap v278: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-15T06:46:04.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:04 smithi089 bash[31372]: cephadm 2024-07-15T06:46:03.190662+0000 mgr.y (mgr.14150) 296 : cephadm [INF] Detected new or changed devices on smithi134 2024-07-15T06:46:04.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:04 smithi089 bash[31372]: audit 2024-07-15T06:46:03.199156+0000 mon.a (mon.0) 563 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:46:04.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:04 smithi089 bash[31372]: audit 2024-07-15T06:46:03.207639+0000 mon.a (mon.0) 564 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:46:04.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:04 smithi089 bash[31372]: audit 2024-07-15T06:46:03.209817+0000 mon.a (mon.0) 565 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:46:04.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:04 smithi089 bash[31372]: audit 2024-07-15T06:46:03.211484+0000 mon.a (mon.0) 566 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:46:04.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:04 smithi089 bash[31372]: audit 2024-07-15T06:46:03.213168+0000 mon.a (mon.0) 567 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:46:04.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:04 smithi089 bash[31372]: cephadm 2024-07-15T06:46:03.214493+0000 mgr.y (mgr.14150) 297 : cephadm [INF] Adjusting osd_memory_target on smithi134 to 5755M 2024-07-15T06:46:04.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:04 smithi089 bash[31372]: audit 2024-07-15T06:46:03.222607+0000 mon.a (mon.0) 568 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:46:04.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:04 smithi089 bash[31372]: audit 2024-07-15T06:46:03.224714+0000 mon.a (mon.0) 569 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:46:04.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:04 smithi089 bash[31372]: audit 2024-07-15T06:46:03.226438+0000 mon.a (mon.0) 570 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:46:04.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:04 smithi089 bash[31372]: audit 2024-07-15T06:46:03.235907+0000 mon.a (mon.0) 571 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:46:06.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:06 smithi134 bash[24068]: cluster 2024-07-15T06:46:04.965141+0000 mgr.y (mgr.14150) 298 : cluster [DBG] pgmap v279: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-15T06:46:06.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:06 smithi089 bash[21793]: cluster 2024-07-15T06:46:04.965141+0000 mgr.y (mgr.14150) 298 : cluster [DBG] pgmap v279: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-15T06:46:06.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:06 smithi089 bash[31372]: cluster 2024-07-15T06:46:04.965141+0000 mgr.y (mgr.14150) 298 : cluster [DBG] pgmap v279: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-15T06:46:07.668 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.b/config 2024-07-15T06:46:08.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:08 smithi134 bash[24068]: cluster 2024-07-15T06:46:06.965799+0000 mgr.y (mgr.14150) 299 : cluster [DBG] pgmap v280: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-15T06:46:08.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:08 smithi089 bash[21793]: cluster 2024-07-15T06:46:06.965799+0000 mgr.y (mgr.14150) 299 : cluster [DBG] pgmap v280: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-15T06:46:08.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:08 smithi089 bash[31372]: cluster 2024-07-15T06:46:06.965799+0000 mgr.y (mgr.14150) 299 : cluster [DBG] pgmap v280: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-15T06:46:10.535 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:10 smithi134 bash[24068]: cluster 2024-07-15T06:46:08.966363+0000 mgr.y (mgr.14150) 300 : cluster [DBG] pgmap v281: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-15T06:46:10.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:10 smithi089 bash[21793]: cluster 2024-07-15T06:46:08.966363+0000 mgr.y (mgr.14150) 300 : cluster [DBG] pgmap v281: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-15T06:46:10.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:10 smithi089 bash[31372]: cluster 2024-07-15T06:46:08.966363+0000 mgr.y (mgr.14150) 300 : cluster [DBG] pgmap v281: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-15T06:46:11.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:11 smithi134 bash[24068]: audit 2024-07-15T06:46:10.863440+0000 mon.a (mon.0) 572 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-07-15T06:46:11.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:11 smithi134 bash[24068]: audit 2024-07-15T06:46:10.869572+0000 mon.a (mon.0) 573 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-07-15T06:46:11.557 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:11 smithi134 bash[24068]: audit 2024-07-15T06:46:10.871145+0000 mon.a (mon.0) 574 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:46:11.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:11 smithi089 bash[21793]: audit 2024-07-15T06:46:10.863440+0000 mon.a (mon.0) 572 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-07-15T06:46:11.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:11 smithi089 bash[21793]: audit 2024-07-15T06:46:10.869572+0000 mon.a (mon.0) 573 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-07-15T06:46:11.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:11 smithi089 bash[21793]: audit 2024-07-15T06:46:10.871145+0000 mon.a (mon.0) 574 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:46:11.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:11 smithi089 bash[31372]: audit 2024-07-15T06:46:10.863440+0000 mon.a (mon.0) 572 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-07-15T06:46:11.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:11 smithi089 bash[31372]: audit 2024-07-15T06:46:10.869572+0000 mon.a (mon.0) 573 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-07-15T06:46:11.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:11 smithi089 bash[31372]: audit 2024-07-15T06:46:10.871145+0000 mon.a (mon.0) 574 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:46:12.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:12 smithi134 bash[24068]: audit 2024-07-15T06:46:10.857166+0000 mgr.y (mgr.14150) 301 : audit [DBG] from='client.24259 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi134:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T06:46:12.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:12 smithi134 bash[24068]: cluster 2024-07-15T06:46:10.966886+0000 mgr.y (mgr.14150) 302 : cluster [DBG] pgmap v282: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-15T06:46:12.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:12 smithi089 bash[21793]: audit 2024-07-15T06:46:10.857166+0000 mgr.y (mgr.14150) 301 : audit [DBG] from='client.24259 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi134:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T06:46:12.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:12 smithi089 bash[21793]: cluster 2024-07-15T06:46:10.966886+0000 mgr.y (mgr.14150) 302 : cluster [DBG] pgmap v282: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-15T06:46:12.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:12 smithi089 bash[31372]: audit 2024-07-15T06:46:10.857166+0000 mgr.y (mgr.14150) 301 : audit [DBG] from='client.24259 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi134:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T06:46:12.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:12 smithi089 bash[31372]: cluster 2024-07-15T06:46:10.966886+0000 mgr.y (mgr.14150) 302 : cluster [DBG] pgmap v282: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-15T06:46:14.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:14 smithi134 bash[24068]: cluster 2024-07-15T06:46:12.967450+0000 mgr.y (mgr.14150) 303 : cluster [DBG] pgmap v283: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-15T06:46:14.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:14 smithi089 bash[21793]: cluster 2024-07-15T06:46:12.967450+0000 mgr.y (mgr.14150) 303 : cluster [DBG] pgmap v283: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-15T06:46:14.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:14 smithi089 bash[31372]: cluster 2024-07-15T06:46:12.967450+0000 mgr.y (mgr.14150) 303 : cluster [DBG] pgmap v283: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-15T06:46:16.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:16 smithi134 bash[24068]: cluster 2024-07-15T06:46:14.967993+0000 mgr.y (mgr.14150) 304 : cluster [DBG] pgmap v284: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-15T06:46:16.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:16 smithi089 bash[21793]: cluster 2024-07-15T06:46:14.967993+0000 mgr.y (mgr.14150) 304 : cluster [DBG] pgmap v284: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-15T06:46:16.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:16 smithi089 bash[31372]: cluster 2024-07-15T06:46:14.967993+0000 mgr.y (mgr.14150) 304 : cluster [DBG] pgmap v284: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-15T06:46:18.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:18 smithi134 bash[24068]: cluster 2024-07-15T06:46:16.968596+0000 mgr.y (mgr.14150) 305 : cluster [DBG] pgmap v285: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-15T06:46:18.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:18 smithi089 bash[21793]: cluster 2024-07-15T06:46:16.968596+0000 mgr.y (mgr.14150) 305 : cluster [DBG] pgmap v285: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-15T06:46:18.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:18 smithi089 bash[31372]: cluster 2024-07-15T06:46:16.968596+0000 mgr.y (mgr.14150) 305 : cluster [DBG] pgmap v285: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-15T06:46:19.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:19 smithi134 bash[24068]: audit 2024-07-15T06:46:19.016551+0000 mon.b (mon.1) 21 : audit [INF] from='client.? 172.21.15.134:0/3839148080' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "87f3a30f-2935-4b92-af37-bd866696838f"}]: dispatch 2024-07-15T06:46:19.557 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:19 smithi134 bash[24068]: audit 2024-07-15T06:46:19.018676+0000 mon.a (mon.0) 575 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "87f3a30f-2935-4b92-af37-bd866696838f"}]: dispatch 2024-07-15T06:46:19.557 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:19 smithi134 bash[24068]: audit 2024-07-15T06:46:19.027605+0000 mon.a (mon.0) 576 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "87f3a30f-2935-4b92-af37-bd866696838f"}]': finished 2024-07-15T06:46:19.557 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:19 smithi134 bash[24068]: cluster 2024-07-15T06:46:19.027712+0000 mon.a (mon.0) 577 : cluster [DBG] osdmap e51: 8 total, 7 up, 8 in 2024-07-15T06:46:19.557 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:19 smithi134 bash[24068]: audit 2024-07-15T06:46:19.027930+0000 mon.a (mon.0) 578 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-07-15T06:46:19.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:19 smithi089 bash[21793]: audit 2024-07-15T06:46:19.016551+0000 mon.b (mon.1) 21 : audit [INF] from='client.? 172.21.15.134:0/3839148080' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "87f3a30f-2935-4b92-af37-bd866696838f"}]: dispatch 2024-07-15T06:46:19.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:19 smithi089 bash[21793]: audit 2024-07-15T06:46:19.018676+0000 mon.a (mon.0) 575 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "87f3a30f-2935-4b92-af37-bd866696838f"}]: dispatch 2024-07-15T06:46:19.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:19 smithi089 bash[21793]: audit 2024-07-15T06:46:19.027605+0000 mon.a (mon.0) 576 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "87f3a30f-2935-4b92-af37-bd866696838f"}]': finished 2024-07-15T06:46:19.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:19 smithi089 bash[21793]: cluster 2024-07-15T06:46:19.027712+0000 mon.a (mon.0) 577 : cluster [DBG] osdmap e51: 8 total, 7 up, 8 in 2024-07-15T06:46:19.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:19 smithi089 bash[21793]: audit 2024-07-15T06:46:19.027930+0000 mon.a (mon.0) 578 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-07-15T06:46:19.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:19 smithi089 bash[31372]: audit 2024-07-15T06:46:19.016551+0000 mon.b (mon.1) 21 : audit [INF] from='client.? 172.21.15.134:0/3839148080' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "87f3a30f-2935-4b92-af37-bd866696838f"}]: dispatch 2024-07-15T06:46:19.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:19 smithi089 bash[31372]: audit 2024-07-15T06:46:19.018676+0000 mon.a (mon.0) 575 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "87f3a30f-2935-4b92-af37-bd866696838f"}]: dispatch 2024-07-15T06:46:19.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:19 smithi089 bash[31372]: audit 2024-07-15T06:46:19.027605+0000 mon.a (mon.0) 576 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "87f3a30f-2935-4b92-af37-bd866696838f"}]': finished 2024-07-15T06:46:19.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:19 smithi089 bash[31372]: cluster 2024-07-15T06:46:19.027712+0000 mon.a (mon.0) 577 : cluster [DBG] osdmap e51: 8 total, 7 up, 8 in 2024-07-15T06:46:19.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:19 smithi089 bash[31372]: audit 2024-07-15T06:46:19.027930+0000 mon.a (mon.0) 578 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-07-15T06:46:20.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:20 smithi134 bash[24068]: cluster 2024-07-15T06:46:18.969200+0000 mgr.y (mgr.14150) 306 : cluster [DBG] pgmap v286: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-15T06:46:20.557 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:20 smithi134 bash[24068]: audit 2024-07-15T06:46:19.659112+0000 mon.b (mon.1) 22 : audit [DBG] from='client.? 172.21.15.134:0/3169617982' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-07-15T06:46:20.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:20 smithi089 bash[21793]: cluster 2024-07-15T06:46:18.969200+0000 mgr.y (mgr.14150) 306 : cluster [DBG] pgmap v286: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-15T06:46:20.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:20 smithi089 bash[21793]: audit 2024-07-15T06:46:19.659112+0000 mon.b (mon.1) 22 : audit [DBG] from='client.? 172.21.15.134:0/3169617982' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-07-15T06:46:20.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:20 smithi089 bash[31372]: cluster 2024-07-15T06:46:18.969200+0000 mgr.y (mgr.14150) 306 : cluster [DBG] pgmap v286: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-15T06:46:20.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:20 smithi089 bash[31372]: audit 2024-07-15T06:46:19.659112+0000 mon.b (mon.1) 22 : audit [DBG] from='client.? 172.21.15.134:0/3169617982' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-07-15T06:46:22.364 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:22 smithi134 bash[24068]: cluster 2024-07-15T06:46:20.969763+0000 mgr.y (mgr.14150) 307 : cluster [DBG] pgmap v288: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-15T06:46:22.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:22 smithi089 bash[21793]: cluster 2024-07-15T06:46:20.969763+0000 mgr.y (mgr.14150) 307 : cluster [DBG] pgmap v288: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-15T06:46:22.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:22 smithi089 bash[31372]: cluster 2024-07-15T06:46:20.969763+0000 mgr.y (mgr.14150) 307 : cluster [DBG] pgmap v288: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-15T06:46:24.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:24 smithi134 bash[24068]: cluster 2024-07-15T06:46:22.970325+0000 mgr.y (mgr.14150) 308 : cluster [DBG] pgmap v289: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-15T06:46:24.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:24 smithi089 bash[21793]: cluster 2024-07-15T06:46:22.970325+0000 mgr.y (mgr.14150) 308 : cluster [DBG] pgmap v289: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-15T06:46:24.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:24 smithi089 bash[31372]: cluster 2024-07-15T06:46:22.970325+0000 mgr.y (mgr.14150) 308 : cluster [DBG] pgmap v289: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-15T06:46:26.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:26 smithi134 bash[24068]: cluster 2024-07-15T06:46:24.970850+0000 mgr.y (mgr.14150) 309 : cluster [DBG] pgmap v290: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-15T06:46:26.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:26 smithi089 bash[21793]: cluster 2024-07-15T06:46:24.970850+0000 mgr.y (mgr.14150) 309 : cluster [DBG] pgmap v290: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-15T06:46:26.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:26 smithi089 bash[31372]: cluster 2024-07-15T06:46:24.970850+0000 mgr.y (mgr.14150) 309 : cluster [DBG] pgmap v290: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-15T06:46:28.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:28 smithi134 bash[24068]: cluster 2024-07-15T06:46:26.971410+0000 mgr.y (mgr.14150) 310 : cluster [DBG] pgmap v291: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-15T06:46:28.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:28 smithi089 bash[21793]: cluster 2024-07-15T06:46:26.971410+0000 mgr.y (mgr.14150) 310 : cluster [DBG] pgmap v291: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-15T06:46:28.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:28 smithi089 bash[31372]: cluster 2024-07-15T06:46:26.971410+0000 mgr.y (mgr.14150) 310 : cluster [DBG] pgmap v291: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-15T06:46:30.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:30 smithi134 bash[24068]: cluster 2024-07-15T06:46:28.971941+0000 mgr.y (mgr.14150) 311 : cluster [DBG] pgmap v292: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-15T06:46:30.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:30 smithi089 bash[21793]: cluster 2024-07-15T06:46:28.971941+0000 mgr.y (mgr.14150) 311 : cluster [DBG] pgmap v292: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-15T06:46:30.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:30 smithi089 bash[31372]: cluster 2024-07-15T06:46:28.971941+0000 mgr.y (mgr.14150) 311 : cluster [DBG] pgmap v292: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-15T06:46:31.541 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:31 smithi134 bash[24068]: audit 2024-07-15T06:46:31.218057+0000 mon.a (mon.0) 579 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-07-15T06:46:31.541 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:31 smithi134 bash[24068]: audit 2024-07-15T06:46:31.220041+0000 mon.a (mon.0) 580 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:46:31.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:31 smithi089 bash[21793]: audit 2024-07-15T06:46:31.218057+0000 mon.a (mon.0) 579 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-07-15T06:46:31.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:31 smithi089 bash[21793]: audit 2024-07-15T06:46:31.220041+0000 mon.a (mon.0) 580 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:46:31.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:31 smithi089 bash[31372]: audit 2024-07-15T06:46:31.218057+0000 mon.a (mon.0) 579 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-07-15T06:46:31.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:31 smithi089 bash[31372]: audit 2024-07-15T06:46:31.220041+0000 mon.a (mon.0) 580 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:46:32.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:32 smithi134 bash[24068]: cluster 2024-07-15T06:46:30.972481+0000 mgr.y (mgr.14150) 312 : cluster [DBG] pgmap v293: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-15T06:46:32.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:32 smithi134 bash[24068]: cephadm 2024-07-15T06:46:31.221727+0000 mgr.y (mgr.14150) 313 : cephadm [INF] Deploying daemon osd.7 on smithi134 2024-07-15T06:46:32.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:32 smithi089 bash[21793]: cluster 2024-07-15T06:46:30.972481+0000 mgr.y (mgr.14150) 312 : cluster [DBG] pgmap v293: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-15T06:46:32.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:32 smithi089 bash[21793]: cephadm 2024-07-15T06:46:31.221727+0000 mgr.y (mgr.14150) 313 : cephadm [INF] Deploying daemon osd.7 on smithi134 2024-07-15T06:46:32.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:32 smithi089 bash[31372]: cluster 2024-07-15T06:46:30.972481+0000 mgr.y (mgr.14150) 312 : cluster [DBG] pgmap v293: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-15T06:46:32.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:32 smithi089 bash[31372]: cephadm 2024-07-15T06:46:31.221727+0000 mgr.y (mgr.14150) 313 : cephadm [INF] Deploying daemon osd.7 on smithi134 2024-07-15T06:46:34.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:34 smithi134 bash[24068]: cluster 2024-07-15T06:46:32.973023+0000 mgr.y (mgr.14150) 314 : cluster [DBG] pgmap v294: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-15T06:46:34.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:34 smithi089 bash[21793]: cluster 2024-07-15T06:46:32.973023+0000 mgr.y (mgr.14150) 314 : cluster [DBG] pgmap v294: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-15T06:46:34.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:34 smithi089 bash[31372]: cluster 2024-07-15T06:46:32.973023+0000 mgr.y (mgr.14150) 314 : cluster [DBG] pgmap v294: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-15T06:46:35.478 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:35 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:46:35.479 INFO:journalctl@ceph.osd.4.smithi134.stdout:Jul 15 06:46:35 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:46:35.479 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:46:35 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:46:35.480 INFO:journalctl@ceph.osd.5.smithi134.stdout:Jul 15 06:46:35 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:46:35.480 INFO:journalctl@ceph.osd.6.smithi134.stdout:Jul 15 06:46:35 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:46:35.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:35 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:46:35.806 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:46:35 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:46:35.807 INFO:journalctl@ceph.osd.4.smithi134.stdout:Jul 15 06:46:35 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:46:35.807 INFO:journalctl@ceph.osd.5.smithi134.stdout:Jul 15 06:46:35 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:46:35.807 INFO:journalctl@ceph.osd.6.smithi134.stdout:Jul 15 06:46:35 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:46:36.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:36 smithi134 bash[24068]: cluster 2024-07-15T06:46:34.973572+0000 mgr.y (mgr.14150) 315 : cluster [DBG] pgmap v295: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-15T06:46:36.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:36 smithi134 bash[24068]: audit 2024-07-15T06:46:35.753959+0000 mon.a (mon.0) 581 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:46:36.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:36 smithi134 bash[24068]: audit 2024-07-15T06:46:35.763136+0000 mon.a (mon.0) 582 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:46:36.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:36 smithi134 bash[24068]: audit 2024-07-15T06:46:35.772363+0000 mon.a (mon.0) 583 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:46:36.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:36 smithi089 bash[21793]: cluster 2024-07-15T06:46:34.973572+0000 mgr.y (mgr.14150) 315 : cluster [DBG] pgmap v295: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-15T06:46:36.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:36 smithi089 bash[21793]: audit 2024-07-15T06:46:35.753959+0000 mon.a (mon.0) 581 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:46:36.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:36 smithi089 bash[21793]: audit 2024-07-15T06:46:35.763136+0000 mon.a (mon.0) 582 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:46:36.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:36 smithi089 bash[21793]: audit 2024-07-15T06:46:35.772363+0000 mon.a (mon.0) 583 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:46:36.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:36 smithi089 bash[31372]: cluster 2024-07-15T06:46:34.973572+0000 mgr.y (mgr.14150) 315 : cluster [DBG] pgmap v295: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-15T06:46:36.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:36 smithi089 bash[31372]: audit 2024-07-15T06:46:35.753959+0000 mon.a (mon.0) 581 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:46:36.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:36 smithi089 bash[31372]: audit 2024-07-15T06:46:35.763136+0000 mon.a (mon.0) 582 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:46:36.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:36 smithi089 bash[31372]: audit 2024-07-15T06:46:35.772363+0000 mon.a (mon.0) 583 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:46:38.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:38 smithi134 bash[24068]: cluster 2024-07-15T06:46:36.974109+0000 mgr.y (mgr.14150) 316 : cluster [DBG] pgmap v296: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-15T06:46:38.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:38 smithi089 bash[21793]: cluster 2024-07-15T06:46:36.974109+0000 mgr.y (mgr.14150) 316 : cluster [DBG] pgmap v296: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-15T06:46:38.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:38 smithi089 bash[31372]: cluster 2024-07-15T06:46:36.974109+0000 mgr.y (mgr.14150) 316 : cluster [DBG] pgmap v296: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-15T06:46:40.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:40 smithi134 bash[24068]: cluster 2024-07-15T06:46:38.974675+0000 mgr.y (mgr.14150) 317 : cluster [DBG] pgmap v297: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-15T06:46:40.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:40 smithi089 bash[21793]: cluster 2024-07-15T06:46:38.974675+0000 mgr.y (mgr.14150) 317 : cluster [DBG] pgmap v297: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-15T06:46:40.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:40 smithi089 bash[31372]: cluster 2024-07-15T06:46:38.974675+0000 mgr.y (mgr.14150) 317 : cluster [DBG] pgmap v297: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-15T06:46:42.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:42 smithi134 bash[24068]: cluster 2024-07-15T06:46:40.975216+0000 mgr.y (mgr.14150) 318 : cluster [DBG] pgmap v298: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-15T06:46:42.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:42 smithi134 bash[24068]: audit 2024-07-15T06:46:41.883856+0000 mon.a (mon.0) 584 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:46:42.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:42 smithi134 bash[24068]: audit 2024-07-15T06:46:41.894948+0000 mon.a (mon.0) 585 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:46:42.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:42 smithi134 bash[24068]: audit 2024-07-15T06:46:41.897120+0000 mon.a (mon.0) 586 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:46:42.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:42 smithi134 bash[24068]: audit 2024-07-15T06:46:41.898857+0000 mon.a (mon.0) 587 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:46:42.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:42 smithi134 bash[24068]: audit 2024-07-15T06:46:41.910734+0000 mon.a (mon.0) 588 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:46:42.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:42 smithi089 bash[21793]: cluster 2024-07-15T06:46:40.975216+0000 mgr.y (mgr.14150) 318 : cluster [DBG] pgmap v298: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-15T06:46:42.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:42 smithi089 bash[21793]: audit 2024-07-15T06:46:41.883856+0000 mon.a (mon.0) 584 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:46:42.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:42 smithi089 bash[21793]: audit 2024-07-15T06:46:41.894948+0000 mon.a (mon.0) 585 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:46:42.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:42 smithi089 bash[21793]: audit 2024-07-15T06:46:41.897120+0000 mon.a (mon.0) 586 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:46:42.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:42 smithi089 bash[21793]: audit 2024-07-15T06:46:41.898857+0000 mon.a (mon.0) 587 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:46:42.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:42 smithi089 bash[21793]: audit 2024-07-15T06:46:41.910734+0000 mon.a (mon.0) 588 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:46:42.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:42 smithi089 bash[31372]: cluster 2024-07-15T06:46:40.975216+0000 mgr.y (mgr.14150) 318 : cluster [DBG] pgmap v298: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-15T06:46:42.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:42 smithi089 bash[31372]: audit 2024-07-15T06:46:41.883856+0000 mon.a (mon.0) 584 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:46:42.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:42 smithi089 bash[31372]: audit 2024-07-15T06:46:41.894948+0000 mon.a (mon.0) 585 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:46:42.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:42 smithi089 bash[31372]: audit 2024-07-15T06:46:41.897120+0000 mon.a (mon.0) 586 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:46:42.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:42 smithi089 bash[31372]: audit 2024-07-15T06:46:41.898857+0000 mon.a (mon.0) 587 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:46:42.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:42 smithi089 bash[31372]: audit 2024-07-15T06:46:41.910734+0000 mon.a (mon.0) 588 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:46:43.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:43 smithi134 bash[24068]: audit 2024-07-15T06:46:42.499648+0000 mon.b (mon.1) 23 : audit [INF] from='osd.7 [v2:172.21.15.134:6824/3883480074,v1:172.21.15.134:6825/3883480074]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-07-15T06:46:43.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:43 smithi134 bash[24068]: audit 2024-07-15T06:46:42.501521+0000 mon.a (mon.0) 589 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-07-15T06:46:43.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:43 smithi089 bash[31372]: audit 2024-07-15T06:46:42.499648+0000 mon.b (mon.1) 23 : audit [INF] from='osd.7 [v2:172.21.15.134:6824/3883480074,v1:172.21.15.134:6825/3883480074]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-07-15T06:46:43.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:43 smithi089 bash[31372]: audit 2024-07-15T06:46:42.501521+0000 mon.a (mon.0) 589 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-07-15T06:46:43.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:43 smithi089 bash[21793]: audit 2024-07-15T06:46:42.499648+0000 mon.b (mon.1) 23 : audit [INF] from='osd.7 [v2:172.21.15.134:6824/3883480074,v1:172.21.15.134:6825/3883480074]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-07-15T06:46:43.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:43 smithi089 bash[21793]: audit 2024-07-15T06:46:42.501521+0000 mon.a (mon.0) 589 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-07-15T06:46:44.335 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:44 smithi089 bash[21793]: cluster 2024-07-15T06:46:42.975733+0000 mgr.y (mgr.14150) 319 : cluster [DBG] pgmap v299: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-15T06:46:44.335 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:44 smithi089 bash[21793]: audit 2024-07-15T06:46:43.310422+0000 mon.a (mon.0) 590 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-07-15T06:46:44.335 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:44 smithi089 bash[21793]: audit 2024-07-15T06:46:43.310521+0000 mon.b (mon.1) 24 : audit [INF] from='osd.7 [v2:172.21.15.134:6824/3883480074,v1:172.21.15.134:6825/3883480074]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]: dispatch 2024-07-15T06:46:44.335 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:44 smithi089 bash[21793]: cluster 2024-07-15T06:46:43.310609+0000 mon.a (mon.0) 591 : cluster [DBG] osdmap e52: 8 total, 7 up, 8 in 2024-07-15T06:46:44.335 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:44 smithi089 bash[21793]: audit 2024-07-15T06:46:43.310876+0000 mon.a (mon.0) 592 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-07-15T06:46:44.335 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:44 smithi089 bash[21793]: audit 2024-07-15T06:46:43.312404+0000 mon.a (mon.0) 593 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]: dispatch 2024-07-15T06:46:44.335 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:44 smithi089 bash[31372]: cluster 2024-07-15T06:46:42.975733+0000 mgr.y (mgr.14150) 319 : cluster [DBG] pgmap v299: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-15T06:46:44.335 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:44 smithi089 bash[31372]: audit 2024-07-15T06:46:43.310422+0000 mon.a (mon.0) 590 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-07-15T06:46:44.336 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:44 smithi089 bash[31372]: audit 2024-07-15T06:46:43.310521+0000 mon.b (mon.1) 24 : audit [INF] from='osd.7 [v2:172.21.15.134:6824/3883480074,v1:172.21.15.134:6825/3883480074]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]: dispatch 2024-07-15T06:46:44.336 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:44 smithi089 bash[31372]: cluster 2024-07-15T06:46:43.310609+0000 mon.a (mon.0) 591 : cluster [DBG] osdmap e52: 8 total, 7 up, 8 in 2024-07-15T06:46:44.336 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:44 smithi089 bash[31372]: audit 2024-07-15T06:46:43.310876+0000 mon.a (mon.0) 592 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-07-15T06:46:44.336 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:44 smithi089 bash[31372]: audit 2024-07-15T06:46:43.312404+0000 mon.a (mon.0) 593 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]: dispatch 2024-07-15T06:46:44.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:44 smithi134 bash[24068]: cluster 2024-07-15T06:46:42.975733+0000 mgr.y (mgr.14150) 319 : cluster [DBG] pgmap v299: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-15T06:46:44.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:44 smithi134 bash[24068]: audit 2024-07-15T06:46:43.310422+0000 mon.a (mon.0) 590 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-07-15T06:46:44.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:44 smithi134 bash[24068]: audit 2024-07-15T06:46:43.310521+0000 mon.b (mon.1) 24 : audit [INF] from='osd.7 [v2:172.21.15.134:6824/3883480074,v1:172.21.15.134:6825/3883480074]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]: dispatch 2024-07-15T06:46:44.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:44 smithi134 bash[24068]: cluster 2024-07-15T06:46:43.310609+0000 mon.a (mon.0) 591 : cluster [DBG] osdmap e52: 8 total, 7 up, 8 in 2024-07-15T06:46:44.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:44 smithi134 bash[24068]: audit 2024-07-15T06:46:43.310876+0000 mon.a (mon.0) 592 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-07-15T06:46:44.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:44 smithi134 bash[24068]: audit 2024-07-15T06:46:43.312404+0000 mon.a (mon.0) 593 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]: dispatch 2024-07-15T06:46:45.313 INFO:teuthology.orchestra.run.smithi134.stdout:Created osd(s) 7 on host 'smithi134' 2024-07-15T06:46:45.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:45 smithi089 bash[31372]: audit 2024-07-15T06:46:44.312770+0000 mon.a (mon.0) 594 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]': finished 2024-07-15T06:46:45.766 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:45 smithi089 bash[31372]: cluster 2024-07-15T06:46:44.312975+0000 mon.a (mon.0) 595 : cluster [DBG] osdmap e53: 8 total, 7 up, 8 in 2024-07-15T06:46:45.767 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:45 smithi089 bash[31372]: audit 2024-07-15T06:46:44.313770+0000 mon.a (mon.0) 596 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-07-15T06:46:45.767 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:45 smithi089 bash[31372]: audit 2024-07-15T06:46:44.323111+0000 mon.a (mon.0) 597 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-07-15T06:46:45.767 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:45 smithi089 bash[31372]: audit 2024-07-15T06:46:45.295639+0000 mon.a (mon.0) 598 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:46:45.767 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:45 smithi089 bash[31372]: audit 2024-07-15T06:46:45.303459+0000 mon.a (mon.0) 599 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:46:45.767 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:45 smithi089 bash[31372]: audit 2024-07-15T06:46:45.312415+0000 mon.a (mon.0) 600 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:46:45.767 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:45 smithi089 bash[21793]: audit 2024-07-15T06:46:44.312770+0000 mon.a (mon.0) 594 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]': finished 2024-07-15T06:46:45.767 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:45 smithi089 bash[21793]: cluster 2024-07-15T06:46:44.312975+0000 mon.a (mon.0) 595 : cluster [DBG] osdmap e53: 8 total, 7 up, 8 in 2024-07-15T06:46:45.767 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:45 smithi089 bash[21793]: audit 2024-07-15T06:46:44.313770+0000 mon.a (mon.0) 596 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-07-15T06:46:45.767 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:45 smithi089 bash[21793]: audit 2024-07-15T06:46:44.323111+0000 mon.a (mon.0) 597 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-07-15T06:46:45.768 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:45 smithi089 bash[21793]: audit 2024-07-15T06:46:45.295639+0000 mon.a (mon.0) 598 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:46:45.768 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:45 smithi089 bash[21793]: audit 2024-07-15T06:46:45.303459+0000 mon.a (mon.0) 599 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:46:45.768 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:45 smithi089 bash[21793]: audit 2024-07-15T06:46:45.312415+0000 mon.a (mon.0) 600 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:46:45.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:45 smithi134 bash[24068]: audit 2024-07-15T06:46:44.312770+0000 mon.a (mon.0) 594 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]': finished 2024-07-15T06:46:45.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:45 smithi134 bash[24068]: cluster 2024-07-15T06:46:44.312975+0000 mon.a (mon.0) 595 : cluster [DBG] osdmap e53: 8 total, 7 up, 8 in 2024-07-15T06:46:45.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:45 smithi134 bash[24068]: audit 2024-07-15T06:46:44.313770+0000 mon.a (mon.0) 596 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-07-15T06:46:45.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:45 smithi134 bash[24068]: audit 2024-07-15T06:46:44.323111+0000 mon.a (mon.0) 597 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-07-15T06:46:45.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:45 smithi134 bash[24068]: audit 2024-07-15T06:46:45.295639+0000 mon.a (mon.0) 598 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:46:45.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:45 smithi134 bash[24068]: audit 2024-07-15T06:46:45.303459+0000 mon.a (mon.0) 599 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:46:45.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:45 smithi134 bash[24068]: audit 2024-07-15T06:46:45.312415+0000 mon.a (mon.0) 600 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:46:45.981 DEBUG:teuthology.orchestra.run.smithi134:osd.7> sudo journalctl -f -n 0 -u ceph-75ba02bc-4274-11ef-bcad-c7b262605968@osd.7.service 2024-07-15T06:46:45.984 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-07-15T06:46:45.984 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph osd stat -f json 2024-07-15T06:46:46.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:46 smithi089 bash[21793]: cluster 2024-07-15T06:46:43.462676+0000 osd.7 (osd.7) 1 : cluster [DBG] purged_snaps scrub starts 2024-07-15T06:46:46.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:46 smithi089 bash[21793]: cluster 2024-07-15T06:46:43.462781+0000 osd.7 (osd.7) 2 : cluster [DBG] purged_snaps scrub ok 2024-07-15T06:46:46.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:46 smithi089 bash[21793]: cluster 2024-07-15T06:46:44.976261+0000 mgr.y (mgr.14150) 320 : cluster [DBG] pgmap v302: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-15T06:46:46.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:46 smithi089 bash[21793]: cluster 2024-07-15T06:46:45.320909+0000 mon.a (mon.0) 601 : cluster [INF] osd.7 [v2:172.21.15.134:6824/3883480074,v1:172.21.15.134:6825/3883480074] boot 2024-07-15T06:46:46.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:46 smithi089 bash[21793]: cluster 2024-07-15T06:46:45.321016+0000 mon.a (mon.0) 602 : cluster [DBG] osdmap e54: 8 total, 8 up, 8 in 2024-07-15T06:46:46.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:46 smithi089 bash[21793]: audit 2024-07-15T06:46:45.322045+0000 mon.a (mon.0) 603 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-07-15T06:46:46.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:46 smithi089 bash[21793]: cluster 2024-07-15T06:46:46.323195+0000 mon.a (mon.0) 604 : cluster [DBG] osdmap e55: 8 total, 8 up, 8 in 2024-07-15T06:46:46.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:46 smithi089 bash[31372]: cluster 2024-07-15T06:46:43.462676+0000 osd.7 (osd.7) 1 : cluster [DBG] purged_snaps scrub starts 2024-07-15T06:46:46.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:46 smithi089 bash[31372]: cluster 2024-07-15T06:46:43.462781+0000 osd.7 (osd.7) 2 : cluster [DBG] purged_snaps scrub ok 2024-07-15T06:46:46.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:46 smithi089 bash[31372]: cluster 2024-07-15T06:46:44.976261+0000 mgr.y (mgr.14150) 320 : cluster [DBG] pgmap v302: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-15T06:46:46.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:46 smithi089 bash[31372]: cluster 2024-07-15T06:46:45.320909+0000 mon.a (mon.0) 601 : cluster [INF] osd.7 [v2:172.21.15.134:6824/3883480074,v1:172.21.15.134:6825/3883480074] boot 2024-07-15T06:46:46.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:46 smithi089 bash[31372]: cluster 2024-07-15T06:46:45.321016+0000 mon.a (mon.0) 602 : cluster [DBG] osdmap e54: 8 total, 8 up, 8 in 2024-07-15T06:46:46.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:46 smithi089 bash[31372]: audit 2024-07-15T06:46:45.322045+0000 mon.a (mon.0) 603 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-07-15T06:46:46.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:46 smithi089 bash[31372]: cluster 2024-07-15T06:46:46.323195+0000 mon.a (mon.0) 604 : cluster [DBG] osdmap e55: 8 total, 8 up, 8 in 2024-07-15T06:46:46.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:46 smithi134 bash[24068]: cluster 2024-07-15T06:46:43.462676+0000 osd.7 (osd.7) 1 : cluster [DBG] purged_snaps scrub starts 2024-07-15T06:46:46.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:46 smithi134 bash[24068]: cluster 2024-07-15T06:46:43.462781+0000 osd.7 (osd.7) 2 : cluster [DBG] purged_snaps scrub ok 2024-07-15T06:46:46.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:46 smithi134 bash[24068]: cluster 2024-07-15T06:46:44.976261+0000 mgr.y (mgr.14150) 320 : cluster [DBG] pgmap v302: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-15T06:46:46.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:46 smithi134 bash[24068]: cluster 2024-07-15T06:46:45.320909+0000 mon.a (mon.0) 601 : cluster [INF] osd.7 [v2:172.21.15.134:6824/3883480074,v1:172.21.15.134:6825/3883480074] boot 2024-07-15T06:46:46.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:46 smithi134 bash[24068]: cluster 2024-07-15T06:46:45.321016+0000 mon.a (mon.0) 602 : cluster [DBG] osdmap e54: 8 total, 8 up, 8 in 2024-07-15T06:46:46.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:46 smithi134 bash[24068]: audit 2024-07-15T06:46:45.322045+0000 mon.a (mon.0) 603 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-07-15T06:46:46.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:46 smithi134 bash[24068]: cluster 2024-07-15T06:46:46.323195+0000 mon.a (mon.0) 604 : cluster [DBG] osdmap e55: 8 total, 8 up, 8 in 2024-07-15T06:46:48.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:48 smithi089 bash[21793]: cluster 2024-07-15T06:46:46.976788+0000 mgr.y (mgr.14150) 321 : cluster [DBG] pgmap v305: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:46:48.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:48 smithi089 bash[21793]: cluster 2024-07-15T06:46:47.324712+0000 mon.a (mon.0) 605 : cluster [DBG] osdmap e56: 8 total, 8 up, 8 in 2024-07-15T06:46:48.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:48 smithi089 bash[31372]: cluster 2024-07-15T06:46:46.976788+0000 mgr.y (mgr.14150) 321 : cluster [DBG] pgmap v305: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:46:48.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:48 smithi089 bash[31372]: cluster 2024-07-15T06:46:47.324712+0000 mon.a (mon.0) 605 : cluster [DBG] osdmap e56: 8 total, 8 up, 8 in 2024-07-15T06:46:48.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:48 smithi134 bash[24068]: cluster 2024-07-15T06:46:46.976788+0000 mgr.y (mgr.14150) 321 : cluster [DBG] pgmap v305: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:46:48.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:48 smithi134 bash[24068]: cluster 2024-07-15T06:46:47.324712+0000 mon.a (mon.0) 605 : cluster [DBG] osdmap e56: 8 total, 8 up, 8 in 2024-07-15T06:46:49.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:49 smithi089 bash[21793]: cluster 2024-07-15T06:46:48.342446+0000 mon.a (mon.0) 606 : cluster [DBG] osdmap e57: 8 total, 8 up, 8 in 2024-07-15T06:46:49.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:49 smithi089 bash[31372]: cluster 2024-07-15T06:46:48.342446+0000 mon.a (mon.0) 606 : cluster [DBG] osdmap e57: 8 total, 8 up, 8 in 2024-07-15T06:46:49.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:49 smithi134 bash[24068]: cluster 2024-07-15T06:46:48.342446+0000 mon.a (mon.0) 606 : cluster [DBG] osdmap e57: 8 total, 8 up, 8 in 2024-07-15T06:46:50.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:50 smithi134 bash[24068]: cluster 2024-07-15T06:46:48.977356+0000 mgr.y (mgr.14150) 322 : cluster [DBG] pgmap v308: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:46:50.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:50 smithi134 bash[24068]: cluster 2024-07-15T06:46:49.350725+0000 mon.a (mon.0) 607 : cluster [DBG] osdmap e58: 8 total, 8 up, 8 in 2024-07-15T06:46:50.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:50 smithi089 bash[21793]: cluster 2024-07-15T06:46:48.977356+0000 mgr.y (mgr.14150) 322 : cluster [DBG] pgmap v308: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:46:50.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:50 smithi089 bash[21793]: cluster 2024-07-15T06:46:49.350725+0000 mon.a (mon.0) 607 : cluster [DBG] osdmap e58: 8 total, 8 up, 8 in 2024-07-15T06:46:50.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:50 smithi089 bash[31372]: cluster 2024-07-15T06:46:48.977356+0000 mgr.y (mgr.14150) 322 : cluster [DBG] pgmap v308: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:46:50.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:50 smithi089 bash[31372]: cluster 2024-07-15T06:46:49.350725+0000 mon.a (mon.0) 607 : cluster [DBG] osdmap e58: 8 total, 8 up, 8 in 2024-07-15T06:46:50.742 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T06:46:52.607 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:52 smithi089 bash[21793]: cluster 2024-07-15T06:46:50.977899+0000 mgr.y (mgr.14150) 323 : cluster [DBG] pgmap v310: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:46:52.608 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:52 smithi089 bash[31372]: cluster 2024-07-15T06:46:50.977899+0000 mgr.y (mgr.14150) 323 : cluster [DBG] pgmap v310: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:46:52.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:52 smithi134 bash[24068]: cluster 2024-07-15T06:46:50.977899+0000 mgr.y (mgr.14150) 323 : cluster [DBG] pgmap v310: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:46:54.408 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-07-15T06:46:54.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:54 smithi134 bash[24068]: cluster 2024-07-15T06:46:52.978551+0000 mgr.y (mgr.14150) 324 : cluster [DBG] pgmap v311: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-07-15T06:46:54.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:54 smithi089 bash[21793]: cluster 2024-07-15T06:46:52.978551+0000 mgr.y (mgr.14150) 324 : cluster [DBG] pgmap v311: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-07-15T06:46:54.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:54 smithi089 bash[31372]: cluster 2024-07-15T06:46:52.978551+0000 mgr.y (mgr.14150) 324 : cluster [DBG] pgmap v311: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-07-15T06:46:55.140 INFO:teuthology.orchestra.run.smithi089.stdout:{"epoch":58,"num_osds":8,"num_up_osds":8,"osd_up_since":1721026005,"num_in_osds":8,"osd_in_since":1721025979,"num_remapped_pgs":0} 2024-07-15T06:46:55.141 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph osd dump --format=json 2024-07-15T06:46:55.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:55 smithi089 bash[21793]: audit 2024-07-15T06:46:54.413123+0000 mon.c (mon.2) 14 : audit [DBG] from='client.? 172.21.15.89:0/716838141' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-07-15T06:46:55.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:55 smithi089 bash[21793]: audit 2024-07-15T06:46:54.643497+0000 mon.a (mon.0) 608 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:46:55.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:55 smithi089 bash[21793]: audit 2024-07-15T06:46:54.653547+0000 mon.a (mon.0) 609 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:46:55.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:55 smithi089 bash[21793]: audit 2024-07-15T06:46:54.655912+0000 mon.a (mon.0) 610 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:46:55.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:55 smithi089 bash[21793]: audit 2024-07-15T06:46:54.657488+0000 mon.a (mon.0) 611 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:46:55.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:55 smithi089 bash[21793]: audit 2024-07-15T06:46:54.658896+0000 mon.a (mon.0) 612 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:46:55.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:55 smithi089 bash[21793]: audit 2024-07-15T06:46:54.660328+0000 mon.a (mon.0) 613 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:46:55.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:55 smithi089 bash[21793]: audit 2024-07-15T06:46:54.669947+0000 mon.a (mon.0) 614 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:46:55.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:55 smithi089 bash[21793]: audit 2024-07-15T06:46:54.672656+0000 mon.a (mon.0) 615 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:46:55.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:55 smithi089 bash[21793]: audit 2024-07-15T06:46:54.674188+0000 mon.a (mon.0) 616 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:46:55.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:55 smithi089 bash[21793]: audit 2024-07-15T06:46:54.685689+0000 mon.a (mon.0) 617 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:46:55.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:55 smithi089 bash[31372]: audit 2024-07-15T06:46:54.413123+0000 mon.c (mon.2) 14 : audit [DBG] from='client.? 172.21.15.89:0/716838141' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-07-15T06:46:55.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:55 smithi089 bash[31372]: audit 2024-07-15T06:46:54.643497+0000 mon.a (mon.0) 608 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:46:55.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:55 smithi089 bash[31372]: audit 2024-07-15T06:46:54.653547+0000 mon.a (mon.0) 609 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:46:55.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:55 smithi089 bash[31372]: audit 2024-07-15T06:46:54.655912+0000 mon.a (mon.0) 610 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:46:55.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:55 smithi089 bash[31372]: audit 2024-07-15T06:46:54.657488+0000 mon.a (mon.0) 611 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:46:55.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:55 smithi089 bash[31372]: audit 2024-07-15T06:46:54.658896+0000 mon.a (mon.0) 612 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:46:55.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:55 smithi089 bash[31372]: audit 2024-07-15T06:46:54.660328+0000 mon.a (mon.0) 613 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:46:55.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:55 smithi089 bash[31372]: audit 2024-07-15T06:46:54.669947+0000 mon.a (mon.0) 614 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:46:55.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:55 smithi089 bash[31372]: audit 2024-07-15T06:46:54.672656+0000 mon.a (mon.0) 615 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:46:55.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:55 smithi089 bash[31372]: audit 2024-07-15T06:46:54.674188+0000 mon.a (mon.0) 616 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:46:55.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:55 smithi089 bash[31372]: audit 2024-07-15T06:46:54.685689+0000 mon.a (mon.0) 617 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:46:55.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:55 smithi134 bash[24068]: audit 2024-07-15T06:46:54.413123+0000 mon.c (mon.2) 14 : audit [DBG] from='client.? 172.21.15.89:0/716838141' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-07-15T06:46:55.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:55 smithi134 bash[24068]: audit 2024-07-15T06:46:54.643497+0000 mon.a (mon.0) 608 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:46:55.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:55 smithi134 bash[24068]: audit 2024-07-15T06:46:54.653547+0000 mon.a (mon.0) 609 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:46:55.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:55 smithi134 bash[24068]: audit 2024-07-15T06:46:54.655912+0000 mon.a (mon.0) 610 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:46:55.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:55 smithi134 bash[24068]: audit 2024-07-15T06:46:54.657488+0000 mon.a (mon.0) 611 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:46:55.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:55 smithi134 bash[24068]: audit 2024-07-15T06:46:54.658896+0000 mon.a (mon.0) 612 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:46:55.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:55 smithi134 bash[24068]: audit 2024-07-15T06:46:54.660328+0000 mon.a (mon.0) 613 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:46:55.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:55 smithi134 bash[24068]: audit 2024-07-15T06:46:54.669947+0000 mon.a (mon.0) 614 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:46:55.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:55 smithi134 bash[24068]: audit 2024-07-15T06:46:54.672656+0000 mon.a (mon.0) 615 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:46:55.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:55 smithi134 bash[24068]: audit 2024-07-15T06:46:54.674188+0000 mon.a (mon.0) 616 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:46:55.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:55 smithi134 bash[24068]: audit 2024-07-15T06:46:54.685689+0000 mon.a (mon.0) 617 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:46:56.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:56 smithi089 bash[21793]: cephadm 2024-07-15T06:46:54.634823+0000 mgr.y (mgr.14150) 325 : cephadm [INF] Detected new or changed devices on smithi134 2024-07-15T06:46:56.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:56 smithi089 bash[21793]: cephadm 2024-07-15T06:46:54.661472+0000 mgr.y (mgr.14150) 326 : cephadm [INF] Adjusting osd_memory_target on smithi134 to 4316M 2024-07-15T06:46:56.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:56 smithi089 bash[21793]: cluster 2024-07-15T06:46:54.979140+0000 mgr.y (mgr.14150) 327 : cluster [DBG] pgmap v312: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 75 KiB/s, 0 objects/s recovering 2024-07-15T06:46:56.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:56 smithi089 bash[31372]: cephadm 2024-07-15T06:46:54.634823+0000 mgr.y (mgr.14150) 325 : cephadm [INF] Detected new or changed devices on smithi134 2024-07-15T06:46:56.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:56 smithi089 bash[31372]: cephadm 2024-07-15T06:46:54.661472+0000 mgr.y (mgr.14150) 326 : cephadm [INF] Adjusting osd_memory_target on smithi134 to 4316M 2024-07-15T06:46:56.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:56 smithi089 bash[31372]: cluster 2024-07-15T06:46:54.979140+0000 mgr.y (mgr.14150) 327 : cluster [DBG] pgmap v312: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 75 KiB/s, 0 objects/s recovering 2024-07-15T06:46:56.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:56 smithi134 bash[24068]: cephadm 2024-07-15T06:46:54.634823+0000 mgr.y (mgr.14150) 325 : cephadm [INF] Detected new or changed devices on smithi134 2024-07-15T06:46:56.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:56 smithi134 bash[24068]: cephadm 2024-07-15T06:46:54.661472+0000 mgr.y (mgr.14150) 326 : cephadm [INF] Adjusting osd_memory_target on smithi134 to 4316M 2024-07-15T06:46:56.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:56 smithi134 bash[24068]: cluster 2024-07-15T06:46:54.979140+0000 mgr.y (mgr.14150) 327 : cluster [DBG] pgmap v312: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 75 KiB/s, 0 objects/s recovering 2024-07-15T06:46:58.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:46:58 smithi089 bash[21793]: cluster 2024-07-15T06:46:56.979717+0000 mgr.y (mgr.14150) 328 : cluster [DBG] pgmap v313: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 67 KiB/s, 0 objects/s recovering 2024-07-15T06:46:58.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:46:58 smithi089 bash[31372]: cluster 2024-07-15T06:46:56.979717+0000 mgr.y (mgr.14150) 328 : cluster [DBG] pgmap v313: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 67 KiB/s, 0 objects/s recovering 2024-07-15T06:46:58.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:46:58 smithi134 bash[24068]: cluster 2024-07-15T06:46:56.979717+0000 mgr.y (mgr.14150) 328 : cluster [DBG] pgmap v313: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 67 KiB/s, 0 objects/s recovering 2024-07-15T06:46:59.900 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T06:47:00.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:00 smithi089 bash[21793]: cluster 2024-07-15T06:46:58.980234+0000 mgr.y (mgr.14150) 329 : cluster [DBG] pgmap v314: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-07-15T06:47:00.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:00 smithi089 bash[31372]: cluster 2024-07-15T06:46:58.980234+0000 mgr.y (mgr.14150) 329 : cluster [DBG] pgmap v314: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-07-15T06:47:00.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:00 smithi134 bash[24068]: cluster 2024-07-15T06:46:58.980234+0000 mgr.y (mgr.14150) 329 : cluster [DBG] pgmap v314: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-07-15T06:47:02.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:02 smithi089 bash[31372]: cluster 2024-07-15T06:47:00.980691+0000 mgr.y (mgr.14150) 330 : cluster [DBG] pgmap v315: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 50 KiB/s, 0 objects/s recovering 2024-07-15T06:47:02.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:02 smithi089 bash[21793]: cluster 2024-07-15T06:47:00.980691+0000 mgr.y (mgr.14150) 330 : cluster [DBG] pgmap v315: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 50 KiB/s, 0 objects/s recovering 2024-07-15T06:47:02.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:02 smithi134 bash[24068]: cluster 2024-07-15T06:47:00.980691+0000 mgr.y (mgr.14150) 330 : cluster [DBG] pgmap v315: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 50 KiB/s, 0 objects/s recovering 2024-07-15T06:47:03.388 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-07-15T06:47:03.388 INFO:teuthology.orchestra.run.smithi089.stdout:{"epoch":58,"fsid":"75ba02bc-4274-11ef-bcad-c7b262605968","created":"2024-07-15T06:37:09.634877+0000","modified":"2024-07-15T06:46:49.337214+0000","last_up_change":"2024-07-15T06:46:45.306680+0000","last_in_change":"2024-07-15T06:46:19.019927+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-07-15T06:42:21.444347+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"79357114-251f-489d-bef5-f6ec37002212","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6802","nonce":843843653},{"type":"v1","addr":"172.21.15.89:6803","nonce":843843653}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6804","nonce":843843653},{"type":"v1","addr":"172.21.15.89:6805","nonce":843843653}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6808","nonce":843843653},{"type":"v1","addr":"172.21.15.89:6809","nonce":843843653}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6806","nonce":843843653},{"type":"v1","addr":"172.21.15.89:6807","nonce":843843653}]},"public_addr":"172.21.15.89:6803/843843653","cluster_addr":"172.21.15.89:6805/843843653","heartbeat_back_addr":"172.21.15.89:6809/843843653","heartbeat_front_addr":"172.21.15.89:6807/843843653","state":["exists","up"]},{"osd":1,"uuid":"d304859e-317a-414e-a16c-f097fc4b2702","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6810","nonce":1536075263},{"type":"v1","addr":"172.21.15.89:6811","nonce":1536075263}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6812","nonce":1536075263},{"type":"v1","addr":"172.21.15.89:6813","nonce":1536075263}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6816","nonce":1536075263},{"type":"v1","addr":"172.21.15.89:6817","nonce":1536075263}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6814","nonce":1536075263},{"type":"v1","addr":"172.21.15.89:6815","nonce":1536075263}]},"public_addr":"172.21.15.89:6811/1536075263","cluster_addr":"172.21.15.89:6813/1536075263","heartbeat_back_addr":"172.21.15.89:6817/1536075263","heartbeat_front_addr":"172.21.15.89:6815/1536075263","state":["exists","up"]},{"osd":2,"uuid":"60cde686-83bb-4ca9-a0f1-6668710e16f1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6818","nonce":3236432318},{"type":"v1","addr":"172.21.15.89:6819","nonce":3236432318}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6820","nonce":3236432318},{"type":"v1","addr":"172.21.15.89:6821","nonce":3236432318}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6824","nonce":3236432318},{"type":"v1","addr":"172.21.15.89:6825","nonce":3236432318}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6822","nonce":3236432318},{"type":"v1","addr":"172.21.15.89:6823","nonce":3236432318}]},"public_addr":"172.21.15.89:6819/3236432318","cluster_addr":"172.21.15.89:6821/3236432318","heartbeat_back_addr":"172.21.15.89:6825/3236432318","heartbeat_front_addr":"172.21.15.89:6823/3236432318","state":["exists","up"]},{"osd":3,"uuid":"8af52bbd-4bc9-4b13-b525-26c92e69c29f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6826","nonce":4083663609},{"type":"v1","addr":"172.21.15.89:6827","nonce":4083663609}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6828","nonce":4083663609},{"type":"v1","addr":"172.21.15.89:6829","nonce":4083663609}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6832","nonce":4083663609},{"type":"v1","addr":"172.21.15.89:6833","nonce":4083663609}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6830","nonce":4083663609},{"type":"v1","addr":"172.21.15.89:6831","nonce":4083663609}]},"public_addr":"172.21.15.89:6827/4083663609","cluster_addr":"172.21.15.89:6829/4083663609","heartbeat_back_addr":"172.21.15.89:6833/4083663609","heartbeat_front_addr":"172.21.15.89:6831/4083663609","state":["exists","up"]},{"osd":4,"uuid":"b97874a6-e72a-4068-9c71-a50b8790be2c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6800","nonce":3588917663},{"type":"v1","addr":"172.21.15.134:6801","nonce":3588917663}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6802","nonce":3588917663},{"type":"v1","addr":"172.21.15.134:6803","nonce":3588917663}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6806","nonce":3588917663},{"type":"v1","addr":"172.21.15.134:6807","nonce":3588917663}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6804","nonce":3588917663},{"type":"v1","addr":"172.21.15.134:6805","nonce":3588917663}]},"public_addr":"172.21.15.134:6801/3588917663","cluster_addr":"172.21.15.134:6803/3588917663","heartbeat_back_addr":"172.21.15.134:6807/3588917663","heartbeat_front_addr":"172.21.15.134:6805/3588917663","state":["exists","up"]},{"osd":5,"uuid":"a834c8ee-e034-423d-ab67-1ab33bc15a97","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":41,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6808","nonce":823577783},{"type":"v1","addr":"172.21.15.134:6809","nonce":823577783}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6810","nonce":823577783},{"type":"v1","addr":"172.21.15.134:6811","nonce":823577783}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6814","nonce":823577783},{"type":"v1","addr":"172.21.15.134:6815","nonce":823577783}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6812","nonce":823577783},{"type":"v1","addr":"172.21.15.134:6813","nonce":823577783}]},"public_addr":"172.21.15.134:6809/823577783","cluster_addr":"172.21.15.134:6811/823577783","heartbeat_back_addr":"172.21.15.134:6815/823577783","heartbeat_front_addr":"172.21.15.134:6813/823577783","state":["exists","up"]},{"osd":6,"uuid":"446cb8fa-8ddb-4f7a-a8a4-c7281e950f74","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6816","nonce":4010812142},{"type":"v1","addr":"172.21.15.134:6817","nonce":4010812142}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6818","nonce":4010812142},{"type":"v1","addr":"172.21.15.134:6819","nonce":4010812142}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6822","nonce":4010812142},{"type":"v1","addr":"172.21.15.134:6823","nonce":4010812142}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6820","nonce":4010812142},{"type":"v1","addr":"172.21.15.134:6821","nonce":4010812142}]},"public_addr":"172.21.15.134:6817/4010812142","cluster_addr":"172.21.15.134:6819/4010812142","heartbeat_back_addr":"172.21.15.134:6823/4010812142","heartbeat_front_addr":"172.21.15.134:6821/4010812142","state":["exists","up"]},{"osd":7,"uuid":"87f3a30f-2935-4b92-af37-bd866696838f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6824","nonce":3883480074},{"type":"v1","addr":"172.21.15.134:6825","nonce":3883480074}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6826","nonce":3883480074},{"type":"v1","addr":"172.21.15.134:6827","nonce":3883480074}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6830","nonce":3883480074},{"type":"v1","addr":"172.21.15.134:6831","nonce":3883480074}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6828","nonce":3883480074},{"type":"v1","addr":"172.21.15.134:6829","nonce":3883480074}]},"public_addr":"172.21.15.134:6825/3883480074","cluster_addr":"172.21.15.134:6827/3883480074","heartbeat_back_addr":"172.21.15.134:6831/3883480074","heartbeat_front_addr":"172.21.15.134:6829/3883480074","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-15T06:40:36.567653+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-15T06:41:26.849575+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-15T06:42:18.731018+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-15T06:43:11.604111+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-15T06:44:03.847505+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-15T06:44:56.895305+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-15T06:45:50.039730+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-15T06:46:43.462786+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.89:6800/2919301358":"2024-07-16T06:38:06.841135+0000","172.21.15.89:6801/2919301358":"2024-07-16T06:38:06.841135+0000","172.21.15.89:0/1672418864":"2024-07-16T06:38:06.841135+0000","172.21.15.89:0/1228522973":"2024-07-16T06:37:35.981306+0000","172.21.15.89:6801/2189595088":"2024-07-16T06:37:35.981306+0000","172.21.15.89:6800/2189595088":"2024-07-16T06:37:35.981306+0000","172.21.15.89:0/2127823469":"2024-07-16T06:37:35.981306+0000","172.21.15.89:0/889381338":"2024-07-16T06:38:06.841135+0000","172.21.15.89:0/1750227302":"2024-07-16T06:38:06.841135+0000","172.21.15.89:0/1877203535":"2024-07-16T06:37:35.981306+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-07-15T06:47:04.075 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-07-15T06:42:21.444347+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2024-07-15T06:47:04.075 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph osd pool get .mgr pg_num 2024-07-15T06:47:04.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:04 smithi089 bash[31372]: cluster 2024-07-15T06:47:02.981261+0000 mgr.y (mgr.14150) 331 : cluster [DBG] pgmap v316: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-07-15T06:47:04.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:04 smithi089 bash[31372]: audit 2024-07-15T06:47:03.391993+0000 mon.a (mon.0) 618 : audit [DBG] from='client.? 172.21.15.89:0/2849020303' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-07-15T06:47:04.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:04 smithi089 bash[21793]: cluster 2024-07-15T06:47:02.981261+0000 mgr.y (mgr.14150) 331 : cluster [DBG] pgmap v316: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-07-15T06:47:04.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:04 smithi089 bash[21793]: audit 2024-07-15T06:47:03.391993+0000 mon.a (mon.0) 618 : audit [DBG] from='client.? 172.21.15.89:0/2849020303' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-07-15T06:47:04.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:04 smithi134 bash[24068]: cluster 2024-07-15T06:47:02.981261+0000 mgr.y (mgr.14150) 331 : cluster [DBG] pgmap v316: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-07-15T06:47:04.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:04 smithi134 bash[24068]: audit 2024-07-15T06:47:03.391993+0000 mon.a (mon.0) 618 : audit [DBG] from='client.? 172.21.15.89:0/2849020303' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-07-15T06:47:06.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:06 smithi134 bash[24068]: cluster 2024-07-15T06:47:04.981822+0000 mgr.y (mgr.14150) 332 : cluster [DBG] pgmap v317: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:47:06.869 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:06 smithi089 bash[21793]: cluster 2024-07-15T06:47:04.981822+0000 mgr.y (mgr.14150) 332 : cluster [DBG] pgmap v317: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:47:06.870 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:06 smithi089 bash[31372]: cluster 2024-07-15T06:47:04.981822+0000 mgr.y (mgr.14150) 332 : cluster [DBG] pgmap v317: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:47:08.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:08 smithi134 bash[24068]: cluster 2024-07-15T06:47:06.982395+0000 mgr.y (mgr.14150) 333 : cluster [DBG] pgmap v318: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:47:08.835 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T06:47:08.869 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:08 smithi089 bash[21793]: cluster 2024-07-15T06:47:06.982395+0000 mgr.y (mgr.14150) 333 : cluster [DBG] pgmap v318: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:47:08.870 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:08 smithi089 bash[31372]: cluster 2024-07-15T06:47:06.982395+0000 mgr.y (mgr.14150) 333 : cluster [DBG] pgmap v318: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:47:10.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:10 smithi134 bash[24068]: cluster 2024-07-15T06:47:08.982960+0000 mgr.y (mgr.14150) 334 : cluster [DBG] pgmap v319: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:47:10.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:10 smithi089 bash[21793]: cluster 2024-07-15T06:47:08.982960+0000 mgr.y (mgr.14150) 334 : cluster [DBG] pgmap v319: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:47:10.870 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:10 smithi089 bash[31372]: cluster 2024-07-15T06:47:08.982960+0000 mgr.y (mgr.14150) 334 : cluster [DBG] pgmap v319: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:47:12.224 INFO:teuthology.orchestra.run.smithi089.stdout:pg_num: 1 2024-07-15T06:47:12.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:12 smithi089 bash[21793]: cluster 2024-07-15T06:47:10.983545+0000 mgr.y (mgr.14150) 335 : cluster [DBG] pgmap v320: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:47:12.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:12 smithi089 bash[21793]: audit 2024-07-15T06:47:12.229247+0000 mon.c (mon.2) 15 : audit [DBG] from='client.? 172.21.15.89:0/193671544' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-07-15T06:47:12.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:12 smithi089 bash[31372]: cluster 2024-07-15T06:47:10.983545+0000 mgr.y (mgr.14150) 335 : cluster [DBG] pgmap v320: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:47:12.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:12 smithi089 bash[31372]: audit 2024-07-15T06:47:12.229247+0000 mon.c (mon.2) 15 : audit [DBG] from='client.? 172.21.15.89:0/193671544' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-07-15T06:47:12.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:12 smithi134 bash[24068]: cluster 2024-07-15T06:47:10.983545+0000 mgr.y (mgr.14150) 335 : cluster [DBG] pgmap v320: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:47:12.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:12 smithi134 bash[24068]: audit 2024-07-15T06:47:12.229247+0000 mon.c (mon.2) 15 : audit [DBG] from='client.? 172.21.15.89:0/193671544' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-07-15T06:47:12.990 INFO:tasks.cephadm:Adding prometheus.a on smithi134 2024-07-15T06:47:12.991 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph orch apply prometheus '1;smithi134=a' 2024-07-15T06:47:14.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:14 smithi134 bash[24068]: cluster 2024-07-15T06:47:12.984066+0000 mgr.y (mgr.14150) 336 : cluster [DBG] pgmap v321: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:47:14.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:14 smithi089 bash[21793]: cluster 2024-07-15T06:47:12.984066+0000 mgr.y (mgr.14150) 336 : cluster [DBG] pgmap v321: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:47:14.870 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:14 smithi089 bash[31372]: cluster 2024-07-15T06:47:12.984066+0000 mgr.y (mgr.14150) 336 : cluster [DBG] pgmap v321: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:47:16.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:16 smithi134 bash[24068]: cluster 2024-07-15T06:47:14.984613+0000 mgr.y (mgr.14150) 337 : cluster [DBG] pgmap v322: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:47:16.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:16 smithi089 bash[21793]: cluster 2024-07-15T06:47:14.984613+0000 mgr.y (mgr.14150) 337 : cluster [DBG] pgmap v322: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:47:16.870 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:16 smithi089 bash[31372]: cluster 2024-07-15T06:47:14.984613+0000 mgr.y (mgr.14150) 337 : cluster [DBG] pgmap v322: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:47:17.740 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.b/config 2024-07-15T06:47:18.797 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:18 smithi134 bash[24068]: cluster 2024-07-15T06:47:16.985159+0000 mgr.y (mgr.14150) 338 : cluster [DBG] pgmap v323: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:47:18.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:18 smithi089 bash[21793]: cluster 2024-07-15T06:47:16.985159+0000 mgr.y (mgr.14150) 338 : cluster [DBG] pgmap v323: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:47:18.870 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:18 smithi089 bash[31372]: cluster 2024-07-15T06:47:16.985159+0000 mgr.y (mgr.14150) 338 : cluster [DBG] pgmap v323: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:47:20.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:20 smithi134 bash[24068]: cluster 2024-07-15T06:47:18.985717+0000 mgr.y (mgr.14150) 339 : cluster [DBG] pgmap v324: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:47:20.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:20 smithi089 bash[21793]: cluster 2024-07-15T06:47:18.985717+0000 mgr.y (mgr.14150) 339 : cluster [DBG] pgmap v324: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:47:20.870 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:20 smithi089 bash[31372]: cluster 2024-07-15T06:47:18.985717+0000 mgr.y (mgr.14150) 339 : cluster [DBG] pgmap v324: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:47:20.934 INFO:teuthology.orchestra.run.smithi134.stdout:Scheduled prometheus update... 2024-07-15T06:47:21.633 DEBUG:teuthology.orchestra.run.smithi134:prometheus.a> sudo journalctl -f -n 0 -u ceph-75ba02bc-4274-11ef-bcad-c7b262605968@prometheus.a.service 2024-07-15T06:47:21.635 INFO:tasks.cephadm:Adding node-exporter.a on smithi089 2024-07-15T06:47:21.635 INFO:tasks.cephadm:Adding node-exporter.b on smithi134 2024-07-15T06:47:21.635 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph orch apply node-exporter '2;smithi089=a;smithi134=b' 2024-07-15T06:47:22.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:21 smithi134 bash[24068]: audit 2024-07-15T06:47:20.928128+0000 mgr.y (mgr.14150) 340 : audit [DBG] from='client.24292 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi134=a", "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T06:47:22.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:21 smithi134 bash[24068]: cephadm 2024-07-15T06:47:20.931021+0000 mgr.y (mgr.14150) 341 : cephadm [INF] Saving service prometheus spec with placement smithi134=a;count:1 2024-07-15T06:47:22.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:21 smithi134 bash[24068]: audit 2024-07-15T06:47:20.937933+0000 mon.a (mon.0) 619 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:47:22.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:21 smithi134 bash[24068]: audit 2024-07-15T06:47:20.939697+0000 mon.a (mon.0) 620 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:47:22.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:21 smithi134 bash[24068]: cluster 2024-07-15T06:47:20.986357+0000 mgr.y (mgr.14150) 342 : cluster [DBG] pgmap v325: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:47:22.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:21 smithi134 bash[24068]: audit 2024-07-15T06:47:21.350490+0000 mon.a (mon.0) 621 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:47:22.307 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:21 smithi134 bash[24068]: audit 2024-07-15T06:47:21.352197+0000 mon.a (mon.0) 622 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:47:22.307 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:21 smithi134 bash[24068]: audit 2024-07-15T06:47:21.363953+0000 mon.a (mon.0) 623 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:47:22.307 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:21 smithi134 bash[24068]: cephadm 2024-07-15T06:47:21.397167+0000 mgr.y (mgr.14150) 343 : cephadm [INF] Deploying daemon prometheus.a on smithi134 2024-07-15T06:47:22.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:21 smithi089 bash[21793]: audit 2024-07-15T06:47:20.928128+0000 mgr.y (mgr.14150) 340 : audit [DBG] from='client.24292 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi134=a", "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T06:47:22.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:21 smithi089 bash[21793]: cephadm 2024-07-15T06:47:20.931021+0000 mgr.y (mgr.14150) 341 : cephadm [INF] Saving service prometheus spec with placement smithi134=a;count:1 2024-07-15T06:47:22.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:21 smithi089 bash[21793]: audit 2024-07-15T06:47:20.937933+0000 mon.a (mon.0) 619 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:47:22.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:21 smithi089 bash[21793]: audit 2024-07-15T06:47:20.939697+0000 mon.a (mon.0) 620 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:47:22.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:21 smithi089 bash[21793]: cluster 2024-07-15T06:47:20.986357+0000 mgr.y (mgr.14150) 342 : cluster [DBG] pgmap v325: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:47:22.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:21 smithi089 bash[21793]: audit 2024-07-15T06:47:21.350490+0000 mon.a (mon.0) 621 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:47:22.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:21 smithi089 bash[21793]: audit 2024-07-15T06:47:21.352197+0000 mon.a (mon.0) 622 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:47:22.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:21 smithi089 bash[21793]: audit 2024-07-15T06:47:21.363953+0000 mon.a (mon.0) 623 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:47:22.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:21 smithi089 bash[21793]: cephadm 2024-07-15T06:47:21.397167+0000 mgr.y (mgr.14150) 343 : cephadm [INF] Deploying daemon prometheus.a on smithi134 2024-07-15T06:47:22.371 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:21 smithi089 bash[31372]: audit 2024-07-15T06:47:20.928128+0000 mgr.y (mgr.14150) 340 : audit [DBG] from='client.24292 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi134=a", "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T06:47:22.371 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:21 smithi089 bash[31372]: cephadm 2024-07-15T06:47:20.931021+0000 mgr.y (mgr.14150) 341 : cephadm [INF] Saving service prometheus spec with placement smithi134=a;count:1 2024-07-15T06:47:22.371 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:21 smithi089 bash[31372]: audit 2024-07-15T06:47:20.937933+0000 mon.a (mon.0) 619 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:47:22.371 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:21 smithi089 bash[31372]: audit 2024-07-15T06:47:20.939697+0000 mon.a (mon.0) 620 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:47:22.371 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:21 smithi089 bash[31372]: cluster 2024-07-15T06:47:20.986357+0000 mgr.y (mgr.14150) 342 : cluster [DBG] pgmap v325: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:47:22.371 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:21 smithi089 bash[31372]: audit 2024-07-15T06:47:21.350490+0000 mon.a (mon.0) 621 : audit [DBG] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:47:22.371 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:21 smithi089 bash[31372]: audit 2024-07-15T06:47:21.352197+0000 mon.a (mon.0) 622 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:47:22.371 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:21 smithi089 bash[31372]: audit 2024-07-15T06:47:21.363953+0000 mon.a (mon.0) 623 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:47:22.371 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:21 smithi089 bash[31372]: cephadm 2024-07-15T06:47:21.397167+0000 mgr.y (mgr.14150) 343 : cephadm [INF] Deploying daemon prometheus.a on smithi134 2024-07-15T06:47:24.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:24 smithi134 bash[24068]: cluster 2024-07-15T06:47:22.986991+0000 mgr.y (mgr.14150) 344 : cluster [DBG] pgmap v326: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:47:24.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:24 smithi089 bash[21793]: cluster 2024-07-15T06:47:22.986991+0000 mgr.y (mgr.14150) 344 : cluster [DBG] pgmap v326: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:47:24.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:24 smithi089 bash[31372]: cluster 2024-07-15T06:47:22.986991+0000 mgr.y (mgr.14150) 344 : cluster [DBG] pgmap v326: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:47:26.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:26 smithi134 bash[24068]: cluster 2024-07-15T06:47:24.987587+0000 mgr.y (mgr.14150) 345 : cluster [DBG] pgmap v327: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:47:26.369 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:26 smithi089 bash[31372]: cluster 2024-07-15T06:47:24.987587+0000 mgr.y (mgr.14150) 345 : cluster [DBG] pgmap v327: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:47:26.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:26 smithi089 bash[21793]: cluster 2024-07-15T06:47:24.987587+0000 mgr.y (mgr.14150) 345 : cluster [DBG] pgmap v327: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:47:26.410 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.b/config 2024-07-15T06:47:28.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:28 smithi134 bash[24068]: cluster 2024-07-15T06:47:26.988156+0000 mgr.y (mgr.14150) 346 : cluster [DBG] pgmap v328: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:47:28.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:28 smithi089 bash[21793]: cluster 2024-07-15T06:47:26.988156+0000 mgr.y (mgr.14150) 346 : cluster [DBG] pgmap v328: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:47:28.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:28 smithi089 bash[31372]: cluster 2024-07-15T06:47:26.988156+0000 mgr.y (mgr.14150) 346 : cluster [DBG] pgmap v328: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:47:30.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:30 smithi134 bash[24068]: cluster 2024-07-15T06:47:28.988717+0000 mgr.y (mgr.14150) 347 : cluster [DBG] pgmap v329: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:47:30.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:30 smithi089 bash[21793]: cluster 2024-07-15T06:47:28.988717+0000 mgr.y (mgr.14150) 347 : cluster [DBG] pgmap v329: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:47:30.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:30 smithi089 bash[31372]: cluster 2024-07-15T06:47:28.988717+0000 mgr.y (mgr.14150) 347 : cluster [DBG] pgmap v329: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:47:32.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:32 smithi134 bash[24068]: cluster 2024-07-15T06:47:30.989260+0000 mgr.y (mgr.14150) 348 : cluster [DBG] pgmap v330: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:47:32.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:32 smithi089 bash[21793]: cluster 2024-07-15T06:47:30.989260+0000 mgr.y (mgr.14150) 348 : cluster [DBG] pgmap v330: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:47:32.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:32 smithi089 bash[31372]: cluster 2024-07-15T06:47:30.989260+0000 mgr.y (mgr.14150) 348 : cluster [DBG] pgmap v330: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:47:32.986 INFO:teuthology.orchestra.run.smithi134.stdout:Scheduled node-exporter update... 2024-07-15T06:47:34.226 DEBUG:teuthology.orchestra.run.smithi089:node-exporter.a> sudo journalctl -f -n 0 -u ceph-75ba02bc-4274-11ef-bcad-c7b262605968@node-exporter.a.service 2024-07-15T06:47:34.229 DEBUG:teuthology.orchestra.run.smithi134:node-exporter.b> sudo journalctl -f -n 0 -u ceph-75ba02bc-4274-11ef-bcad-c7b262605968@node-exporter.b.service 2024-07-15T06:47:34.232 INFO:tasks.cephadm:Adding alertmanager.a on smithi089 2024-07-15T06:47:34.232 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph orch apply alertmanager '1;smithi089=a' 2024-07-15T06:47:34.238 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:33 smithi089 bash[21793]: audit 2024-07-15T06:47:32.979860+0000 mgr.y (mgr.14150) 349 : audit [DBG] from='client.24298 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi089=a;smithi134=b", "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T06:47:34.239 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:33 smithi089 bash[21793]: cephadm 2024-07-15T06:47:32.982856+0000 mgr.y (mgr.14150) 350 : cephadm [INF] Saving service node-exporter spec with placement smithi089=a;smithi134=b;count:2 2024-07-15T06:47:34.239 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:33 smithi089 bash[21793]: cluster 2024-07-15T06:47:32.989812+0000 mgr.y (mgr.14150) 351 : cluster [DBG] pgmap v331: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:47:34.239 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:33 smithi089 bash[21793]: audit 2024-07-15T06:47:32.990139+0000 mon.a (mon.0) 624 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:47:34.239 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:33 smithi089 bash[31372]: audit 2024-07-15T06:47:32.979860+0000 mgr.y (mgr.14150) 349 : audit [DBG] from='client.24298 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi089=a;smithi134=b", "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T06:47:34.239 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:33 smithi089 bash[31372]: cephadm 2024-07-15T06:47:32.982856+0000 mgr.y (mgr.14150) 350 : cephadm [INF] Saving service node-exporter spec with placement smithi089=a;smithi134=b;count:2 2024-07-15T06:47:34.239 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:33 smithi089 bash[31372]: cluster 2024-07-15T06:47:32.989812+0000 mgr.y (mgr.14150) 351 : cluster [DBG] pgmap v331: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:47:34.239 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:33 smithi089 bash[31372]: audit 2024-07-15T06:47:32.990139+0000 mon.a (mon.0) 624 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:47:34.240 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:33 smithi134 bash[24068]: audit 2024-07-15T06:47:32.979860+0000 mgr.y (mgr.14150) 349 : audit [DBG] from='client.24298 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi089=a;smithi134=b", "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T06:47:34.240 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:33 smithi134 bash[24068]: cephadm 2024-07-15T06:47:32.982856+0000 mgr.y (mgr.14150) 350 : cephadm [INF] Saving service node-exporter spec with placement smithi089=a;smithi134=b;count:2 2024-07-15T06:47:34.240 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:33 smithi134 bash[24068]: cluster 2024-07-15T06:47:32.989812+0000 mgr.y (mgr.14150) 351 : cluster [DBG] pgmap v331: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:47:34.240 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:33 smithi134 bash[24068]: audit 2024-07-15T06:47:32.990139+0000 mon.a (mon.0) 624 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:47:36.294 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:35 smithi134 bash[24068]: cluster 2024-07-15T06:47:34.990412+0000 mgr.y (mgr.14150) 352 : cluster [DBG] pgmap v332: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:47:36.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:35 smithi089 bash[21793]: cluster 2024-07-15T06:47:34.990412+0000 mgr.y (mgr.14150) 352 : cluster [DBG] pgmap v332: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:47:36.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:35 smithi089 bash[31372]: cluster 2024-07-15T06:47:34.990412+0000 mgr.y (mgr.14150) 352 : cluster [DBG] pgmap v332: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:47:37.382 INFO:journalctl@ceph.osd.4.smithi134.stdout:Jul 15 06:47:37 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:47:37.382 INFO:journalctl@ceph.osd.5.smithi134.stdout:Jul 15 06:47:37 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:47:37.382 INFO:journalctl@ceph.osd.6.smithi134.stdout:Jul 15 06:47:37 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:47:37.383 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:37 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:47:37.383 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:47:37 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:47:37.383 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 06:47:37 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:47:37.645 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:47:37 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:47:37.645 INFO:journalctl@ceph.osd.4.smithi134.stdout:Jul 15 06:47:37 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:47:37.645 INFO:journalctl@ceph.osd.6.smithi134.stdout:Jul 15 06:47:37 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:47:37.645 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:37 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:47:37.646 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 06:47:37 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:47:37.646 INFO:journalctl@ceph.osd.5.smithi134.stdout:Jul 15 06:47:37 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:47:37.646 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:47:37 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:47:37.647 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:47:37 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:47:37.647 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:47:37 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:47:37.647 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:47:37 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:47:37.647 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:47:37 smithi134 systemd[1]: Started Ceph prometheus.a for 75ba02bc-4274-11ef-bcad-c7b262605968. 2024-07-15T06:47:38.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:38 smithi134 bash[24068]: cluster 2024-07-15T06:47:36.990990+0000 mgr.y (mgr.14150) 353 : cluster [DBG] pgmap v333: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:47:38.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:38 smithi134 bash[24068]: audit 2024-07-15T06:47:37.685755+0000 mon.a (mon.0) 625 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:47:38.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:38 smithi134 bash[24068]: audit 2024-07-15T06:47:37.695181+0000 mon.a (mon.0) 626 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:47:38.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:38 smithi134 bash[24068]: audit 2024-07-15T06:47:37.703328+0000 mon.a (mon.0) 627 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:47:38.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:38 smithi134 bash[24068]: audit 2024-07-15T06:47:37.710244+0000 mon.a (mon.0) 628 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-07-15T06:47:38.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:38 smithi089 bash[21793]: cluster 2024-07-15T06:47:36.990990+0000 mgr.y (mgr.14150) 353 : cluster [DBG] pgmap v333: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:47:38.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:38 smithi089 bash[21793]: audit 2024-07-15T06:47:37.685755+0000 mon.a (mon.0) 625 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:47:38.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:38 smithi089 bash[21793]: audit 2024-07-15T06:47:37.695181+0000 mon.a (mon.0) 626 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:47:38.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:38 smithi089 bash[21793]: audit 2024-07-15T06:47:37.703328+0000 mon.a (mon.0) 627 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:47:38.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:38 smithi089 bash[21793]: audit 2024-07-15T06:47:37.710244+0000 mon.a (mon.0) 628 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-07-15T06:47:38.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:38 smithi089 bash[31372]: cluster 2024-07-15T06:47:36.990990+0000 mgr.y (mgr.14150) 353 : cluster [DBG] pgmap v333: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:47:38.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:38 smithi089 bash[31372]: audit 2024-07-15T06:47:37.685755+0000 mon.a (mon.0) 625 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:47:38.371 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:38 smithi089 bash[31372]: audit 2024-07-15T06:47:37.695181+0000 mon.a (mon.0) 626 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:47:38.371 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:38 smithi089 bash[31372]: audit 2024-07-15T06:47:37.703328+0000 mon.a (mon.0) 627 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:47:38.371 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:38 smithi089 bash[31372]: audit 2024-07-15T06:47:37.710244+0000 mon.a (mon.0) 628 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-07-15T06:47:39.029 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:47:38 smithi089 bash[22045]: ignoring --setuser ceph since I am not root 2024-07-15T06:47:39.029 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:47:38 smithi089 bash[22045]: ignoring --setgroup ceph since I am not root 2024-07-15T06:47:39.029 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:47:38 smithi089 bash[22045]: debug 2024-07-15T06:47:38.756+0000 7fc70e943640 1 -- 172.21.15.89:0/3226401797 <== mon.2 v2:172.21.15.89:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55f4228d49c0 con 0x55f4228e0800 2024-07-15T06:47:39.029 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:47:38 smithi089 bash[22045]: debug 2024-07-15T06:47:38.864+0000 7fc71119dec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-07-15T06:47:39.030 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:47:38 smithi134 bash[25019]: ignoring --setuser ceph since I am not root 2024-07-15T06:47:39.030 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:47:38 smithi134 bash[25019]: ignoring --setgroup ceph since I am not root 2024-07-15T06:47:39.030 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:47:38 smithi134 bash[25019]: debug 2024-07-15T06:47:38.764+0000 7fcdc1d0b640 1 -- 172.21.15.134:0/562957036 <== mon.1 v2:172.21.15.134:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55f2e9cd49c0 con 0x55f2e9ce0c00 2024-07-15T06:47:39.030 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:47:38 smithi134 bash[25019]: debug 2024-07-15T06:47:38.884+0000 7fcdc4565ec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-07-15T06:47:39.030 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:47:38 smithi134 bash[48372]: ts=2024-07-15T06:47:38.850Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-07-15T06:47:39.030 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:47:38 smithi134 bash[48372]: ts=2024-07-15T06:47:38.850Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-07-15T06:47:39.031 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:47:38 smithi134 bash[48372]: ts=2024-07-15T06:47:38.850Z caller=main.go:570 level=info host_details="(Linux 5.15.0-113-generic #123-Ubuntu SMP Mon Jun 10 08:16:17 UTC 2024 x86_64 smithi134 (none))" 2024-07-15T06:47:39.031 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:47:38 smithi134 bash[48372]: ts=2024-07-15T06:47:38.850Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-07-15T06:47:39.031 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:47:38 smithi134 bash[48372]: ts=2024-07-15T06:47:38.850Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-07-15T06:47:39.031 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:47:38 smithi134 bash[48372]: ts=2024-07-15T06:47:38.851Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-07-15T06:47:39.031 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:47:38 smithi134 bash[48372]: ts=2024-07-15T06:47:38.852Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-07-15T06:47:39.031 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:47:38 smithi134 bash[48372]: ts=2024-07-15T06:47:38.852Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-07-15T06:47:39.031 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:47:38 smithi134 bash[48372]: ts=2024-07-15T06:47:38.852Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-07-15T06:47:39.031 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:47:38 smithi134 bash[48372]: ts=2024-07-15T06:47:38.857Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-07-15T06:47:39.031 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:47:38 smithi134 bash[48372]: ts=2024-07-15T06:47:38.857Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=4.513µs 2024-07-15T06:47:39.031 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:47:38 smithi134 bash[48372]: ts=2024-07-15T06:47:38.857Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-07-15T06:47:39.031 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:47:38 smithi134 bash[48372]: ts=2024-07-15T06:47:38.857Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=0 2024-07-15T06:47:39.031 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:47:38 smithi134 bash[48372]: ts=2024-07-15T06:47:38.857Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=29.844µs wal_replay_duration=341.846µs wbl_replay_duration=87ns total_replay_duration=415.631µs 2024-07-15T06:47:39.031 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:47:38 smithi134 bash[48372]: ts=2024-07-15T06:47:38.858Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-07-15T06:47:39.032 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:47:38 smithi134 bash[48372]: ts=2024-07-15T06:47:38.858Z caller=main.go:1029 level=info msg="TSDB started" 2024-07-15T06:47:39.032 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:47:38 smithi134 bash[48372]: ts=2024-07-15T06:47:38.858Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-07-15T06:47:39.032 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:47:38 smithi134 bash[48372]: ts=2024-07-15T06:47:38.876Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=17.40109ms db_storage=901ns remote_storage=1.046µs web_handler=403ns query_engine=701ns scrape=581.908µs scrape_sd=19.076µs notify=797ns notify_sd=1.378µs rules=16.593871ms tracing=5.946µs 2024-07-15T06:47:39.032 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:47:38 smithi134 bash[48372]: ts=2024-07-15T06:47:38.876Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-07-15T06:47:39.032 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:47:38 smithi134 bash[48372]: ts=2024-07-15T06:47:38.876Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-07-15T06:47:39.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:39 smithi134 bash[24068]: audit 2024-07-15T06:47:38.035640+0000 mon.a (mon.0) 629 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:47:39.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:39 smithi134 bash[24068]: audit 2024-07-15T06:47:38.714263+0000 mon.a (mon.0) 630 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-07-15T06:47:39.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:39 smithi134 bash[24068]: cluster 2024-07-15T06:47:38.714312+0000 mon.a (mon.0) 631 : cluster [DBG] mgrmap e16: y(active, since 9m), standbys: x 2024-07-15T06:47:39.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:39 smithi089 bash[21793]: audit 2024-07-15T06:47:38.035640+0000 mon.a (mon.0) 629 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:47:39.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:39 smithi089 bash[21793]: audit 2024-07-15T06:47:38.714263+0000 mon.a (mon.0) 630 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-07-15T06:47:39.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:39 smithi089 bash[21793]: cluster 2024-07-15T06:47:38.714312+0000 mon.a (mon.0) 631 : cluster [DBG] mgrmap e16: y(active, since 9m), standbys: x 2024-07-15T06:47:39.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:39 smithi089 bash[31372]: audit 2024-07-15T06:47:38.035640+0000 mon.a (mon.0) 629 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' 2024-07-15T06:47:39.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:39 smithi089 bash[31372]: audit 2024-07-15T06:47:38.714263+0000 mon.a (mon.0) 630 : audit [INF] from='mgr.14150 172.21.15.89:0/1674511066' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-07-15T06:47:39.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:39 smithi089 bash[31372]: cluster 2024-07-15T06:47:38.714312+0000 mon.a (mon.0) 631 : cluster [DBG] mgrmap e16: y(active, since 9m), standbys: x 2024-07-15T06:47:40.370 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:47:40 smithi089 bash[22045]: debug 2024-07-15T06:47:40.068+0000 7fc71119dec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-07-15T06:47:40.370 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:47:40 smithi089 bash[22045]: debug 2024-07-15T06:47:40.132+0000 7fc71119dec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-07-15T06:47:40.370 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:47:40 smithi089 bash[22045]: debug 2024-07-15T06:47:40.284+0000 7fc71119dec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-07-15T06:47:40.555 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:47:40 smithi134 bash[25019]: debug 2024-07-15T06:47:40.084+0000 7fcdc4565ec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-07-15T06:47:40.555 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:47:40 smithi134 bash[25019]: debug 2024-07-15T06:47:40.148+0000 7fcdc4565ec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-07-15T06:47:40.556 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:47:40 smithi134 bash[25019]: debug 2024-07-15T06:47:40.300+0000 7fcdc4565ec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-07-15T06:47:41.103 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.b/config 2024-07-15T06:47:41.214 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:47:40 smithi089 bash[22045]: debug 2024-07-15T06:47:40.916+0000 7fc71119dec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-07-15T06:47:41.215 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:47:40 smithi089 bash[22045]: debug 2024-07-15T06:47:40.988+0000 7fc71119dec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-07-15T06:47:41.215 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:47:41 smithi089 bash[22045]: debug 2024-07-15T06:47:41.056+0000 7fc71119dec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-07-15T06:47:41.220 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:47:40 smithi134 bash[25019]: debug 2024-07-15T06:47:40.932+0000 7fcdc4565ec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-07-15T06:47:41.220 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:47:41 smithi134 bash[25019]: debug 2024-07-15T06:47:41.000+0000 7fcdc4565ec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-07-15T06:47:41.220 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:47:41 smithi134 bash[25019]: debug 2024-07-15T06:47:41.068+0000 7fcdc4565ec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-07-15T06:47:41.555 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:47:41 smithi134 bash[25019]: debug 2024-07-15T06:47:41.232+0000 7fcdc4565ec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-07-15T06:47:41.555 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:47:41 smithi134 bash[25019]: debug 2024-07-15T06:47:41.308+0000 7fcdc4565ec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-07-15T06:47:41.556 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:47:41 smithi134 bash[25019]: debug 2024-07-15T06:47:41.376+0000 7fcdc4565ec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-07-15T06:47:41.589 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:47:41 smithi089 bash[22045]: debug 2024-07-15T06:47:41.216+0000 7fc71119dec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-07-15T06:47:41.589 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:47:41 smithi089 bash[22045]: debug 2024-07-15T06:47:41.292+0000 7fc71119dec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-07-15T06:47:41.590 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:47:41 smithi089 bash[22045]: debug 2024-07-15T06:47:41.360+0000 7fc71119dec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-07-15T06:47:41.590 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:47:41 smithi089 bash[22045]: debug 2024-07-15T06:47:41.588+0000 7fc71119dec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-07-15T06:47:41.869 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:47:41 smithi089 bash[22045]: debug 2024-07-15T06:47:41.656+0000 7fc71119dec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-07-15T06:47:41.870 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:47:41 smithi089 bash[22045]: debug 2024-07-15T06:47:41.828+0000 7fc71119dec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-07-15T06:47:41.925 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:47:41 smithi134 bash[25019]: debug 2024-07-15T06:47:41.608+0000 7fcdc4565ec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-07-15T06:47:41.925 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:47:41 smithi134 bash[25019]: debug 2024-07-15T06:47:41.676+0000 7fcdc4565ec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-07-15T06:47:41.926 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:47:41 smithi134 bash[25019]: debug 2024-07-15T06:47:41.852+0000 7fcdc4565ec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-07-15T06:47:42.303 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:47:41 smithi089 bash[22045]: debug 2024-07-15T06:47:41.904+0000 7fc71119dec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-07-15T06:47:42.303 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:47:42 smithi089 bash[22045]: debug 2024-07-15T06:47:42.140+0000 7fc71119dec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-07-15T06:47:42.305 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:47:41 smithi134 bash[25019]: debug 2024-07-15T06:47:41.928+0000 7fcdc4565ec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-07-15T06:47:42.305 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:47:42 smithi134 bash[25019]: debug 2024-07-15T06:47:42.164+0000 7fcdc4565ec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-07-15T06:47:42.620 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:47:42 smithi089 bash[22045]: debug 2024-07-15T06:47:42.304+0000 7fc71119dec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-07-15T06:47:42.620 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:47:42 smithi089 bash[22045]: debug 2024-07-15T06:47:42.380+0000 7fc71119dec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-07-15T06:47:42.620 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:47:42 smithi089 bash[22045]: debug 2024-07-15T06:47:42.536+0000 7fc71119dec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-07-15T06:47:42.686 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:47:42 smithi134 bash[25019]: debug 2024-07-15T06:47:42.332+0000 7fcdc4565ec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-07-15T06:47:42.686 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:47:42 smithi134 bash[25019]: debug 2024-07-15T06:47:42.416+0000 7fcdc4565ec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-07-15T06:47:42.686 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:47:42 smithi134 bash[25019]: debug 2024-07-15T06:47:42.576+0000 7fcdc4565ec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-07-15T06:47:43.555 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:47:43 smithi134 bash[25019]: debug 2024-07-15T06:47:43.348+0000 7fcdc4565ec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-07-15T06:47:43.555 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:47:43 smithi134 bash[25019]: debug 2024-07-15T06:47:43.424+0000 7fcdc4565ec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-07-15T06:47:43.593 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:47:43 smithi089 bash[22045]: debug 2024-07-15T06:47:43.316+0000 7fc71119dec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-07-15T06:47:43.593 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:47:43 smithi089 bash[22045]: debug 2024-07-15T06:47:43.392+0000 7fc71119dec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-07-15T06:47:43.593 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:47:43 smithi089 bash[22045]: debug 2024-07-15T06:47:43.592+0000 7fc71119dec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-07-15T06:47:43.869 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:47:43 smithi089 bash[22045]: debug 2024-07-15T06:47:43.668+0000 7fc71119dec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-07-15T06:47:44.021 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:47:43 smithi134 bash[25019]: debug 2024-07-15T06:47:43.628+0000 7fcdc4565ec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-07-15T06:47:44.021 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:47:43 smithi134 bash[25019]: debug 2024-07-15T06:47:43.700+0000 7fcdc4565ec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-07-15T06:47:44.295 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:47:43 smithi089 bash[22045]: debug 2024-07-15T06:47:43.980+0000 7fc71119dec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-07-15T06:47:44.295 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:47:44 smithi089 bash[22045]: debug 2024-07-15T06:47:44.060+0000 7fc71119dec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-07-15T06:47:44.295 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:47:44 smithi089 bash[22045]: [15/Jul/2024:06:47:44] ENGINE Bus STARTING 2024-07-15T06:47:44.295 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:47:44 smithi089 bash[22045]: CherryPy Checker: 2024-07-15T06:47:44.295 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:47:44 smithi089 bash[22045]: The Application mounted at '' has an empty config. 2024-07-15T06:47:44.296 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:44 smithi089 bash[31372]: cluster 2024-07-15T06:47:44.086430+0000 mon.a (mon.0) 632 : cluster [INF] Active manager daemon y restarted 2024-07-15T06:47:44.296 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:44 smithi089 bash[31372]: cluster 2024-07-15T06:47:44.086798+0000 mon.a (mon.0) 633 : cluster [INF] Activating manager daemon y 2024-07-15T06:47:44.296 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:44 smithi089 bash[31372]: cluster 2024-07-15T06:47:44.098066+0000 mon.a (mon.0) 634 : cluster [DBG] osdmap e59: 8 total, 8 up, 8 in 2024-07-15T06:47:44.296 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:44 smithi089 bash[31372]: cluster 2024-07-15T06:47:44.098823+0000 mon.a (mon.0) 635 : cluster [DBG] mgrmap e17: y(active, starting, since 0.0121405s), standbys: x 2024-07-15T06:47:44.296 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:44 smithi089 bash[31372]: audit 2024-07-15T06:47:44.100821+0000 mon.a (mon.0) 636 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-07-15T06:47:44.296 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:44 smithi089 bash[31372]: audit 2024-07-15T06:47:44.101106+0000 mon.a (mon.0) 637 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-15T06:47:44.296 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:44 smithi089 bash[31372]: audit 2024-07-15T06:47:44.101383+0000 mon.a (mon.0) 638 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-15T06:47:44.296 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:44 smithi089 bash[31372]: audit 2024-07-15T06:47:44.102207+0000 mon.a (mon.0) 639 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-07-15T06:47:44.296 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:44 smithi089 bash[31372]: audit 2024-07-15T06:47:44.102580+0000 mon.a (mon.0) 640 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-07-15T06:47:44.296 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:44 smithi089 bash[31372]: audit 2024-07-15T06:47:44.102912+0000 mon.a (mon.0) 641 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-07-15T06:47:44.297 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:44 smithi089 bash[31372]: audit 2024-07-15T06:47:44.103211+0000 mon.a (mon.0) 642 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-07-15T06:47:44.297 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:44 smithi089 bash[31372]: audit 2024-07-15T06:47:44.103524+0000 mon.a (mon.0) 643 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-07-15T06:47:44.297 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:44 smithi089 bash[31372]: audit 2024-07-15T06:47:44.103829+0000 mon.a (mon.0) 644 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-07-15T06:47:44.297 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:44 smithi089 bash[31372]: audit 2024-07-15T06:47:44.104117+0000 mon.a (mon.0) 645 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-07-15T06:47:44.297 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:44 smithi089 bash[31372]: audit 2024-07-15T06:47:44.104439+0000 mon.a (mon.0) 646 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-07-15T06:47:44.297 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:44 smithi089 bash[31372]: audit 2024-07-15T06:47:44.104666+0000 mon.a (mon.0) 647 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-07-15T06:47:44.297 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:44 smithi089 bash[31372]: audit 2024-07-15T06:47:44.104985+0000 mon.a (mon.0) 648 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-07-15T06:47:44.297 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:44 smithi089 bash[31372]: audit 2024-07-15T06:47:44.105239+0000 mon.a (mon.0) 649 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-07-15T06:47:44.297 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:44 smithi089 bash[31372]: audit 2024-07-15T06:47:44.105438+0000 mon.a (mon.0) 650 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-07-15T06:47:44.297 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:44 smithi089 bash[21793]: cluster 2024-07-15T06:47:44.086430+0000 mon.a (mon.0) 632 : cluster [INF] Active manager daemon y restarted 2024-07-15T06:47:44.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:44 smithi089 bash[21793]: cluster 2024-07-15T06:47:44.086798+0000 mon.a (mon.0) 633 : cluster [INF] Activating manager daemon y 2024-07-15T06:47:44.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:44 smithi089 bash[21793]: cluster 2024-07-15T06:47:44.098066+0000 mon.a (mon.0) 634 : cluster [DBG] osdmap e59: 8 total, 8 up, 8 in 2024-07-15T06:47:44.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:44 smithi089 bash[21793]: cluster 2024-07-15T06:47:44.098823+0000 mon.a (mon.0) 635 : cluster [DBG] mgrmap e17: y(active, starting, since 0.0121405s), standbys: x 2024-07-15T06:47:44.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:44 smithi089 bash[21793]: audit 2024-07-15T06:47:44.100821+0000 mon.a (mon.0) 636 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-07-15T06:47:44.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:44 smithi089 bash[21793]: audit 2024-07-15T06:47:44.101106+0000 mon.a (mon.0) 637 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-15T06:47:44.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:44 smithi089 bash[21793]: audit 2024-07-15T06:47:44.101383+0000 mon.a (mon.0) 638 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-15T06:47:44.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:44 smithi089 bash[21793]: audit 2024-07-15T06:47:44.102207+0000 mon.a (mon.0) 639 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-07-15T06:47:44.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:44 smithi089 bash[21793]: audit 2024-07-15T06:47:44.102580+0000 mon.a (mon.0) 640 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-07-15T06:47:44.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:44 smithi089 bash[21793]: audit 2024-07-15T06:47:44.102912+0000 mon.a (mon.0) 641 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-07-15T06:47:44.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:44 smithi089 bash[21793]: audit 2024-07-15T06:47:44.103211+0000 mon.a (mon.0) 642 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-07-15T06:47:44.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:44 smithi089 bash[21793]: audit 2024-07-15T06:47:44.103524+0000 mon.a (mon.0) 643 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-07-15T06:47:44.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:44 smithi089 bash[21793]: audit 2024-07-15T06:47:44.103829+0000 mon.a (mon.0) 644 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-07-15T06:47:44.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:44 smithi089 bash[21793]: audit 2024-07-15T06:47:44.104117+0000 mon.a (mon.0) 645 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-07-15T06:47:44.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:44 smithi089 bash[21793]: audit 2024-07-15T06:47:44.104439+0000 mon.a (mon.0) 646 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-07-15T06:47:44.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:44 smithi089 bash[21793]: audit 2024-07-15T06:47:44.104666+0000 mon.a (mon.0) 647 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-07-15T06:47:44.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:44 smithi089 bash[21793]: audit 2024-07-15T06:47:44.104985+0000 mon.a (mon.0) 648 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-07-15T06:47:44.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:44 smithi089 bash[21793]: audit 2024-07-15T06:47:44.105239+0000 mon.a (mon.0) 649 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-07-15T06:47:44.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:44 smithi089 bash[21793]: audit 2024-07-15T06:47:44.105438+0000 mon.a (mon.0) 650 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-07-15T06:47:44.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:44 smithi089 bash[31372]: audit 2024-07-15T06:47:44.105978+0000 mon.a (mon.0) 651 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-07-15T06:47:44.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:44 smithi089 bash[31372]: cluster 2024-07-15T06:47:44.132124+0000 mon.a (mon.0) 652 : cluster [DBG] Standby manager daemon x restarted 2024-07-15T06:47:44.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:44 smithi089 bash[31372]: cluster 2024-07-15T06:47:44.132276+0000 mon.a (mon.0) 653 : cluster [DBG] Standby manager daemon x started 2024-07-15T06:47:44.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:44 smithi089 bash[31372]: audit 2024-07-15T06:47:44.132808+0000 mon.b (mon.1) 25 : audit [DBG] from='mgr.? 172.21.15.134:0/2699563848' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-07-15T06:47:44.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:44 smithi089 bash[31372]: cluster 2024-07-15T06:47:44.133351+0000 mon.a (mon.0) 654 : cluster [INF] Manager daemon y is now available 2024-07-15T06:47:44.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:44 smithi089 bash[31372]: audit 2024-07-15T06:47:44.133685+0000 mon.b (mon.1) 26 : audit [DBG] from='mgr.? 172.21.15.134:0/2699563848' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-07-15T06:47:44.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:44 smithi089 bash[31372]: audit 2024-07-15T06:47:44.135378+0000 mon.b (mon.1) 27 : audit [DBG] from='mgr.? 172.21.15.134:0/2699563848' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-07-15T06:47:44.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:44 smithi089 bash[31372]: audit 2024-07-15T06:47:44.136382+0000 mon.b (mon.1) 28 : audit [DBG] from='mgr.? 172.21.15.134:0/2699563848' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-07-15T06:47:44.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:44 smithi089 bash[21793]: audit 2024-07-15T06:47:44.105978+0000 mon.a (mon.0) 651 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-07-15T06:47:44.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:44 smithi089 bash[21793]: cluster 2024-07-15T06:47:44.132124+0000 mon.a (mon.0) 652 : cluster [DBG] Standby manager daemon x restarted 2024-07-15T06:47:44.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:44 smithi089 bash[21793]: cluster 2024-07-15T06:47:44.132276+0000 mon.a (mon.0) 653 : cluster [DBG] Standby manager daemon x started 2024-07-15T06:47:44.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:44 smithi089 bash[21793]: audit 2024-07-15T06:47:44.132808+0000 mon.b (mon.1) 25 : audit [DBG] from='mgr.? 172.21.15.134:0/2699563848' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-07-15T06:47:44.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:44 smithi089 bash[21793]: cluster 2024-07-15T06:47:44.133351+0000 mon.a (mon.0) 654 : cluster [INF] Manager daemon y is now available 2024-07-15T06:47:44.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:44 smithi089 bash[21793]: audit 2024-07-15T06:47:44.133685+0000 mon.b (mon.1) 26 : audit [DBG] from='mgr.? 172.21.15.134:0/2699563848' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-07-15T06:47:44.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:44 smithi089 bash[21793]: audit 2024-07-15T06:47:44.135378+0000 mon.b (mon.1) 27 : audit [DBG] from='mgr.? 172.21.15.134:0/2699563848' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-07-15T06:47:44.301 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:44 smithi089 bash[21793]: audit 2024-07-15T06:47:44.136382+0000 mon.b (mon.1) 28 : audit [DBG] from='mgr.? 172.21.15.134:0/2699563848' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-07-15T06:47:44.305 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:47:44 smithi134 bash[25019]: debug 2024-07-15T06:47:44.024+0000 7fcdc4565ec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-07-15T06:47:44.305 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:47:44 smithi134 bash[25019]: debug 2024-07-15T06:47:44.108+0000 7fcdc4565ec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-07-15T06:47:44.306 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:47:44 smithi134 bash[25019]: [15/Jul/2024:06:47:44] ENGINE Bus STARTING 2024-07-15T06:47:44.306 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:47:44 smithi134 bash[25019]: CherryPy Checker: 2024-07-15T06:47:44.306 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:47:44 smithi134 bash[25019]: The Application mounted at '' has an empty config. 2024-07-15T06:47:44.306 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:47:44 smithi134 bash[25019]: [15/Jul/2024:06:47:44] ENGINE Serving on http://:::9283 2024-07-15T06:47:44.306 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:47:44 smithi134 bash[25019]: [15/Jul/2024:06:47:44] ENGINE Bus STARTED 2024-07-15T06:47:44.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:44 smithi134 bash[24068]: cluster 2024-07-15T06:47:44.086430+0000 mon.a (mon.0) 632 : cluster [INF] Active manager daemon y restarted 2024-07-15T06:47:44.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:44 smithi134 bash[24068]: cluster 2024-07-15T06:47:44.086798+0000 mon.a (mon.0) 633 : cluster [INF] Activating manager daemon y 2024-07-15T06:47:44.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:44 smithi134 bash[24068]: cluster 2024-07-15T06:47:44.098066+0000 mon.a (mon.0) 634 : cluster [DBG] osdmap e59: 8 total, 8 up, 8 in 2024-07-15T06:47:44.307 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:44 smithi134 bash[24068]: cluster 2024-07-15T06:47:44.098823+0000 mon.a (mon.0) 635 : cluster [DBG] mgrmap e17: y(active, starting, since 0.0121405s), standbys: x 2024-07-15T06:47:44.307 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:44 smithi134 bash[24068]: audit 2024-07-15T06:47:44.100821+0000 mon.a (mon.0) 636 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-07-15T06:47:44.307 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:44 smithi134 bash[24068]: audit 2024-07-15T06:47:44.101106+0000 mon.a (mon.0) 637 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-15T06:47:44.307 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:44 smithi134 bash[24068]: audit 2024-07-15T06:47:44.101383+0000 mon.a (mon.0) 638 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-15T06:47:44.307 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:44 smithi134 bash[24068]: audit 2024-07-15T06:47:44.102207+0000 mon.a (mon.0) 639 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-07-15T06:47:44.307 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:44 smithi134 bash[24068]: audit 2024-07-15T06:47:44.102580+0000 mon.a (mon.0) 640 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-07-15T06:47:44.307 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:44 smithi134 bash[24068]: audit 2024-07-15T06:47:44.102912+0000 mon.a (mon.0) 641 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-07-15T06:47:44.307 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:44 smithi134 bash[24068]: audit 2024-07-15T06:47:44.103211+0000 mon.a (mon.0) 642 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-07-15T06:47:44.307 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:44 smithi134 bash[24068]: audit 2024-07-15T06:47:44.103524+0000 mon.a (mon.0) 643 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-07-15T06:47:44.307 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:44 smithi134 bash[24068]: audit 2024-07-15T06:47:44.103829+0000 mon.a (mon.0) 644 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-07-15T06:47:44.307 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:44 smithi134 bash[24068]: audit 2024-07-15T06:47:44.104117+0000 mon.a (mon.0) 645 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-07-15T06:47:44.307 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:44 smithi134 bash[24068]: audit 2024-07-15T06:47:44.104439+0000 mon.a (mon.0) 646 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-07-15T06:47:44.307 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:44 smithi134 bash[24068]: audit 2024-07-15T06:47:44.104666+0000 mon.a (mon.0) 647 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-07-15T06:47:44.308 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:44 smithi134 bash[24068]: audit 2024-07-15T06:47:44.104985+0000 mon.a (mon.0) 648 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-07-15T06:47:44.308 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:44 smithi134 bash[24068]: audit 2024-07-15T06:47:44.105239+0000 mon.a (mon.0) 649 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-07-15T06:47:44.308 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:44 smithi134 bash[24068]: audit 2024-07-15T06:47:44.105438+0000 mon.a (mon.0) 650 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-07-15T06:47:44.308 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:44 smithi134 bash[24068]: audit 2024-07-15T06:47:44.105978+0000 mon.a (mon.0) 651 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-07-15T06:47:44.308 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:44 smithi134 bash[24068]: cluster 2024-07-15T06:47:44.132124+0000 mon.a (mon.0) 652 : cluster [DBG] Standby manager daemon x restarted 2024-07-15T06:47:44.308 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:44 smithi134 bash[24068]: cluster 2024-07-15T06:47:44.132276+0000 mon.a (mon.0) 653 : cluster [DBG] Standby manager daemon x started 2024-07-15T06:47:44.308 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:44 smithi134 bash[24068]: audit 2024-07-15T06:47:44.132808+0000 mon.b (mon.1) 25 : audit [DBG] from='mgr.? 172.21.15.134:0/2699563848' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-07-15T06:47:44.308 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:44 smithi134 bash[24068]: cluster 2024-07-15T06:47:44.133351+0000 mon.a (mon.0) 654 : cluster [INF] Manager daemon y is now available 2024-07-15T06:47:44.308 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:44 smithi134 bash[24068]: audit 2024-07-15T06:47:44.133685+0000 mon.b (mon.1) 26 : audit [DBG] from='mgr.? 172.21.15.134:0/2699563848' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-07-15T06:47:44.308 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:44 smithi134 bash[24068]: audit 2024-07-15T06:47:44.135378+0000 mon.b (mon.1) 27 : audit [DBG] from='mgr.? 172.21.15.134:0/2699563848' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-07-15T06:47:44.308 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:44 smithi134 bash[24068]: audit 2024-07-15T06:47:44.136382+0000 mon.b (mon.1) 28 : audit [DBG] from='mgr.? 172.21.15.134:0/2699563848' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-07-15T06:47:44.619 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:47:44 smithi089 bash[22045]: [15/Jul/2024:06:47:44] ENGINE Serving on http://:::9283 2024-07-15T06:47:44.620 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:47:44 smithi089 bash[22045]: [15/Jul/2024:06:47:44] ENGINE Bus STARTED 2024-07-15T06:47:45.129 INFO:teuthology.orchestra.run.smithi134.stdout:Scheduled alertmanager update... 2024-07-15T06:47:45.437 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:45 smithi134 bash[24068]: audit 2024-07-15T06:47:44.153973+0000 mon.a (mon.0) 655 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:47:45.437 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:45 smithi134 bash[24068]: audit 2024-07-15T06:47:44.155802+0000 mon.a (mon.0) 656 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:47:45.437 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:45 smithi134 bash[24068]: audit 2024-07-15T06:47:44.179029+0000 mon.a (mon.0) 657 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:47:45.437 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:45 smithi134 bash[24068]: audit 2024-07-15T06:47:44.187460+0000 mon.a (mon.0) 658 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:47:45.437 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:45 smithi134 bash[24068]: audit 2024-07-15T06:47:44.192769+0000 mon.a (mon.0) 659 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-07-15T06:47:45.437 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:45 smithi134 bash[24068]: audit 2024-07-15T06:47:44.203700+0000 mon.a (mon.0) 660 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-07-15T06:47:45.437 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:45 smithi134 bash[24068]: cluster 2024-07-15T06:47:45.108372+0000 mon.a (mon.0) 661 : cluster [DBG] mgrmap e18: y(active, since 1.02167s), standbys: x 2024-07-15T06:47:45.437 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:45 smithi134 bash[24068]: audit 2024-07-15T06:47:45.132795+0000 mon.a (mon.0) 662 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:47:45.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:45 smithi089 bash[31372]: audit 2024-07-15T06:47:44.153973+0000 mon.a (mon.0) 655 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:47:45.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:45 smithi089 bash[31372]: audit 2024-07-15T06:47:44.155802+0000 mon.a (mon.0) 656 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:47:45.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:45 smithi089 bash[31372]: audit 2024-07-15T06:47:44.179029+0000 mon.a (mon.0) 657 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:47:45.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:45 smithi089 bash[31372]: audit 2024-07-15T06:47:44.187460+0000 mon.a (mon.0) 658 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:47:45.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:45 smithi089 bash[31372]: audit 2024-07-15T06:47:44.192769+0000 mon.a (mon.0) 659 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-07-15T06:47:45.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:45 smithi089 bash[31372]: audit 2024-07-15T06:47:44.203700+0000 mon.a (mon.0) 660 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-07-15T06:47:45.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:45 smithi089 bash[31372]: cluster 2024-07-15T06:47:45.108372+0000 mon.a (mon.0) 661 : cluster [DBG] mgrmap e18: y(active, since 1.02167s), standbys: x 2024-07-15T06:47:45.533 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:45 smithi089 bash[31372]: audit 2024-07-15T06:47:45.132795+0000 mon.a (mon.0) 662 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:47:45.533 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:45 smithi089 bash[21793]: audit 2024-07-15T06:47:44.153973+0000 mon.a (mon.0) 655 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:47:45.533 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:45 smithi089 bash[21793]: audit 2024-07-15T06:47:44.155802+0000 mon.a (mon.0) 656 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:47:45.533 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:45 smithi089 bash[21793]: audit 2024-07-15T06:47:44.179029+0000 mon.a (mon.0) 657 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:47:45.533 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:45 smithi089 bash[21793]: audit 2024-07-15T06:47:44.187460+0000 mon.a (mon.0) 658 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:47:45.533 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:45 smithi089 bash[21793]: audit 2024-07-15T06:47:44.192769+0000 mon.a (mon.0) 659 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-07-15T06:47:45.533 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:45 smithi089 bash[21793]: audit 2024-07-15T06:47:44.203700+0000 mon.a (mon.0) 660 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-07-15T06:47:45.533 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:45 smithi089 bash[21793]: cluster 2024-07-15T06:47:45.108372+0000 mon.a (mon.0) 661 : cluster [DBG] mgrmap e18: y(active, since 1.02167s), standbys: x 2024-07-15T06:47:45.533 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:45 smithi089 bash[21793]: audit 2024-07-15T06:47:45.132795+0000 mon.a (mon.0) 662 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:47:45.870 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:47:45 smithi089 bash[22045]: debug 2024-07-15T06:47:45.532+0000 7fc6b3fe9640 -1 log_channel(cephadm) log [ERR] : [15/Jul/2024:06:47:45] ENGINE Error in HTTPServer.serve 2024-07-15T06:47:45.870 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:47:45 smithi089 bash[22045]: Traceback (most recent call last): 2024-07-15T06:47:45.870 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:47:45 smithi089 bash[22045]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-07-15T06:47:45.870 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:47:45 smithi089 bash[22045]: self._connections.run(self.expiration_interval) 2024-07-15T06:47:45.871 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:47:45 smithi089 bash[22045]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-07-15T06:47:45.871 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:47:45 smithi089 bash[22045]: self._run(expiration_interval) 2024-07-15T06:47:45.871 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:47:45 smithi089 bash[22045]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-07-15T06:47:45.871 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:47:45 smithi089 bash[22045]: new_conn = self._from_server_socket(self.server.socket) 2024-07-15T06:47:45.871 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:47:45 smithi089 bash[22045]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-07-15T06:47:45.871 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:47:45 smithi089 bash[22045]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-07-15T06:47:45.871 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:47:45 smithi089 bash[22045]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-07-15T06:47:45.871 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:47:45 smithi089 bash[22045]: s = self.context.wrap_socket( 2024-07-15T06:47:45.871 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:47:45 smithi089 bash[22045]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-07-15T06:47:45.871 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:47:45 smithi089 bash[22045]: return self.sslsocket_class._create( 2024-07-15T06:47:45.871 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:47:45 smithi089 bash[22045]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-07-15T06:47:45.871 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:47:45 smithi089 bash[22045]: self.do_handshake() 2024-07-15T06:47:45.871 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:47:45 smithi089 bash[22045]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-07-15T06:47:45.872 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:47:45 smithi089 bash[22045]: self._sslobj.do_handshake() 2024-07-15T06:47:45.872 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:47:45 smithi089 bash[22045]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-07-15T06:47:45.901 DEBUG:teuthology.orchestra.run.smithi089:alertmanager.a> sudo journalctl -f -n 0 -u ceph-75ba02bc-4274-11ef-bcad-c7b262605968@alertmanager.a.service 2024-07-15T06:47:45.904 INFO:tasks.cephadm:Adding grafana.a on smithi134 2024-07-15T06:47:45.904 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph orch apply grafana '1;smithi134=a' 2024-07-15T06:47:46.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:46 smithi134 bash[24068]: cephadm 2024-07-15T06:47:45.124508+0000 mgr.y (mgr.24272) 3 : cephadm [INF] Saving service alertmanager spec with placement smithi089=a;count:1 2024-07-15T06:47:46.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:46 smithi134 bash[24068]: cephadm 2024-07-15T06:47:45.415952+0000 mgr.y (mgr.24272) 4 : cephadm [INF] [15/Jul/2024:06:47:45] ENGINE Bus STARTING 2024-07-15T06:47:46.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:46 smithi134 bash[24068]: cephadm 2024-07-15T06:47:45.535180+0000 mgr.y (mgr.24272) 5 : cephadm [INF] [15/Jul/2024:06:47:45] ENGINE Serving on https://172.21.15.89:7150 2024-07-15T06:47:46.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:46 smithi134 bash[24068]: cephadm 2024-07-15T06:47:45.535321+0000 mgr.y (mgr.24272) 6 : cephadm [INF] [15/Jul/2024:06:47:45] ENGINE Bus STARTED 2024-07-15T06:47:46.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:46 smithi134 bash[24068]: cephadm 2024-07-15T06:47:45.537163+0000 mgr.y (mgr.24272) 7 : cephadm [ERR] [15/Jul/2024:06:47:45] ENGINE Error in HTTPServer.serve 2024-07-15T06:47:46.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:46 smithi134 bash[24068]: Traceback (most recent call last): 2024-07-15T06:47:46.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:46 smithi134 bash[24068]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-07-15T06:47:46.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:46 smithi134 bash[24068]: self._connections.run(self.expiration_interval) 2024-07-15T06:47:46.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:46 smithi134 bash[24068]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-07-15T06:47:46.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:46 smithi134 bash[24068]: self._run(expiration_interval) 2024-07-15T06:47:46.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:46 smithi134 bash[24068]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-07-15T06:47:46.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:46 smithi134 bash[24068]: new_conn = self._from_server_socket(self.server.socket) 2024-07-15T06:47:46.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:46 smithi134 bash[24068]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-07-15T06:47:46.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:46 smithi134 bash[24068]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-07-15T06:47:46.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:46 smithi134 bash[24068]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-07-15T06:47:46.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:46 smithi134 bash[24068]: s = self.context.wrap_socket( 2024-07-15T06:47:46.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:46 smithi134 bash[24068]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-07-15T06:47:46.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:46 smithi134 bash[24068]: return self.sslsocket_class._create( 2024-07-15T06:47:46.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:46 smithi134 bash[24068]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-07-15T06:47:46.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:46 smithi134 bash[24068]: self.do_handshake() 2024-07-15T06:47:46.808 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:46 smithi134 bash[24068]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-07-15T06:47:46.808 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:46 smithi134 bash[24068]: self._sslobj.do_handshake() 2024-07-15T06:47:46.808 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:46 smithi134 bash[24068]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-07-15T06:47:46.808 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:46 smithi134 bash[24068]: audit 2024-07-15T06:47:45.544825+0000 mon.a (mon.0) 663 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:47:46.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:46 smithi089 bash[21793]: cephadm 2024-07-15T06:47:45.124508+0000 mgr.y (mgr.24272) 3 : cephadm [INF] Saving service alertmanager spec with placement smithi089=a;count:1 2024-07-15T06:47:46.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:46 smithi089 bash[21793]: cephadm 2024-07-15T06:47:45.415952+0000 mgr.y (mgr.24272) 4 : cephadm [INF] [15/Jul/2024:06:47:45] ENGINE Bus STARTING 2024-07-15T06:47:46.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:46 smithi089 bash[21793]: cephadm 2024-07-15T06:47:45.535180+0000 mgr.y (mgr.24272) 5 : cephadm [INF] [15/Jul/2024:06:47:45] ENGINE Serving on https://172.21.15.89:7150 2024-07-15T06:47:46.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:46 smithi089 bash[21793]: cephadm 2024-07-15T06:47:45.535321+0000 mgr.y (mgr.24272) 6 : cephadm [INF] [15/Jul/2024:06:47:45] ENGINE Bus STARTED 2024-07-15T06:47:46.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:46 smithi089 bash[21793]: cephadm 2024-07-15T06:47:45.537163+0000 mgr.y (mgr.24272) 7 : cephadm [ERR] [15/Jul/2024:06:47:45] ENGINE Error in HTTPServer.serve 2024-07-15T06:47:46.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:46 smithi089 bash[21793]: Traceback (most recent call last): 2024-07-15T06:47:46.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:46 smithi089 bash[21793]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-07-15T06:47:46.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:46 smithi089 bash[21793]: self._connections.run(self.expiration_interval) 2024-07-15T06:47:46.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:46 smithi089 bash[21793]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-07-15T06:47:46.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:46 smithi089 bash[21793]: self._run(expiration_interval) 2024-07-15T06:47:46.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:46 smithi089 bash[21793]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-07-15T06:47:46.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:46 smithi089 bash[21793]: new_conn = self._from_server_socket(self.server.socket) 2024-07-15T06:47:46.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:46 smithi089 bash[21793]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-07-15T06:47:46.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:46 smithi089 bash[21793]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-07-15T06:47:46.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:46 smithi089 bash[21793]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-07-15T06:47:46.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:46 smithi089 bash[21793]: s = self.context.wrap_socket( 2024-07-15T06:47:46.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:46 smithi089 bash[21793]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-07-15T06:47:46.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:46 smithi089 bash[21793]: return self.sslsocket_class._create( 2024-07-15T06:47:46.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:46 smithi089 bash[21793]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-07-15T06:47:46.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:46 smithi089 bash[21793]: self.do_handshake() 2024-07-15T06:47:46.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:46 smithi089 bash[21793]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-07-15T06:47:46.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:46 smithi089 bash[21793]: self._sslobj.do_handshake() 2024-07-15T06:47:46.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:46 smithi089 bash[21793]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-07-15T06:47:46.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:46 smithi089 bash[21793]: audit 2024-07-15T06:47:45.544825+0000 mon.a (mon.0) 663 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:47:46.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:46 smithi089 bash[31372]: cephadm 2024-07-15T06:47:45.124508+0000 mgr.y (mgr.24272) 3 : cephadm [INF] Saving service alertmanager spec with placement smithi089=a;count:1 2024-07-15T06:47:46.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:46 smithi089 bash[31372]: cephadm 2024-07-15T06:47:45.415952+0000 mgr.y (mgr.24272) 4 : cephadm [INF] [15/Jul/2024:06:47:45] ENGINE Bus STARTING 2024-07-15T06:47:46.873 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:46 smithi089 bash[31372]: cephadm 2024-07-15T06:47:45.535180+0000 mgr.y (mgr.24272) 5 : cephadm [INF] [15/Jul/2024:06:47:45] ENGINE Serving on https://172.21.15.89:7150 2024-07-15T06:47:46.873 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:46 smithi089 bash[31372]: cephadm 2024-07-15T06:47:45.535321+0000 mgr.y (mgr.24272) 6 : cephadm [INF] [15/Jul/2024:06:47:45] ENGINE Bus STARTED 2024-07-15T06:47:46.873 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:46 smithi089 bash[31372]: cephadm 2024-07-15T06:47:45.537163+0000 mgr.y (mgr.24272) 7 : cephadm [ERR] [15/Jul/2024:06:47:45] ENGINE Error in HTTPServer.serve 2024-07-15T06:47:46.873 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:46 smithi089 bash[31372]: Traceback (most recent call last): 2024-07-15T06:47:46.873 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:46 smithi089 bash[31372]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-07-15T06:47:46.874 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:46 smithi089 bash[31372]: self._connections.run(self.expiration_interval) 2024-07-15T06:47:46.874 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:46 smithi089 bash[31372]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-07-15T06:47:46.874 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:46 smithi089 bash[31372]: self._run(expiration_interval) 2024-07-15T06:47:46.874 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:46 smithi089 bash[31372]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-07-15T06:47:46.874 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:46 smithi089 bash[31372]: new_conn = self._from_server_socket(self.server.socket) 2024-07-15T06:47:46.874 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:46 smithi089 bash[31372]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-07-15T06:47:46.874 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:46 smithi089 bash[31372]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-07-15T06:47:46.874 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:46 smithi089 bash[31372]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-07-15T06:47:46.874 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:46 smithi089 bash[31372]: s = self.context.wrap_socket( 2024-07-15T06:47:46.874 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:46 smithi089 bash[31372]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-07-15T06:47:46.874 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:46 smithi089 bash[31372]: return self.sslsocket_class._create( 2024-07-15T06:47:46.874 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:46 smithi089 bash[31372]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-07-15T06:47:46.875 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:46 smithi089 bash[31372]: self.do_handshake() 2024-07-15T06:47:46.875 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:46 smithi089 bash[31372]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-07-15T06:47:46.875 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:46 smithi089 bash[31372]: self._sslobj.do_handshake() 2024-07-15T06:47:46.875 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:46 smithi089 bash[31372]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-07-15T06:47:46.875 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:46 smithi089 bash[31372]: audit 2024-07-15T06:47:45.544825+0000 mon.a (mon.0) 663 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:47:47.305 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:47:46 smithi134 bash[25019]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:47:46] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-07-15T06:47:47.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:47 smithi134 bash[24068]: cluster 2024-07-15T06:47:46.103739+0000 mgr.y (mgr.24272) 8 : cluster [DBG] pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:47:47.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:47 smithi134 bash[24068]: cluster 2024-07-15T06:47:47.119570+0000 mon.a (mon.0) 664 : cluster [DBG] mgrmap e19: y(active, since 3s), standbys: x 2024-07-15T06:47:47.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:47 smithi089 bash[21793]: cluster 2024-07-15T06:47:46.103739+0000 mgr.y (mgr.24272) 8 : cluster [DBG] pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:47:47.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:47 smithi089 bash[21793]: cluster 2024-07-15T06:47:47.119570+0000 mon.a (mon.0) 664 : cluster [DBG] mgrmap e19: y(active, since 3s), standbys: x 2024-07-15T06:47:47.870 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:47 smithi089 bash[31372]: cluster 2024-07-15T06:47:46.103739+0000 mgr.y (mgr.24272) 8 : cluster [DBG] pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:47:47.870 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:47 smithi089 bash[31372]: cluster 2024-07-15T06:47:47.119570+0000 mon.a (mon.0) 664 : cluster [DBG] mgrmap e19: y(active, since 3s), standbys: x 2024-07-15T06:47:49.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:49 smithi134 bash[24068]: cluster 2024-07-15T06:47:48.104267+0000 mgr.y (mgr.24272) 9 : cluster [DBG] pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:47:49.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:49 smithi134 bash[24068]: cluster 2024-07-15T06:47:48.562201+0000 mon.a (mon.0) 665 : cluster [DBG] mgrmap e20: y(active, since 4s), standbys: x 2024-07-15T06:47:49.869 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:49 smithi089 bash[21793]: cluster 2024-07-15T06:47:48.104267+0000 mgr.y (mgr.24272) 9 : cluster [DBG] pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:47:49.869 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:49 smithi089 bash[21793]: cluster 2024-07-15T06:47:48.562201+0000 mon.a (mon.0) 665 : cluster [DBG] mgrmap e20: y(active, since 4s), standbys: x 2024-07-15T06:47:49.870 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:49 smithi089 bash[31372]: cluster 2024-07-15T06:47:48.104267+0000 mgr.y (mgr.24272) 9 : cluster [DBG] pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:47:49.870 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:49 smithi089 bash[31372]: cluster 2024-07-15T06:47:48.562201+0000 mon.a (mon.0) 665 : cluster [DBG] mgrmap e20: y(active, since 4s), standbys: x 2024-07-15T06:47:50.432 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.b/config 2024-07-15T06:47:50.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:50 smithi134 bash[24068]: audit 2024-07-15T06:47:49.607827+0000 mon.a (mon.0) 666 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:47:50.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:50 smithi134 bash[24068]: audit 2024-07-15T06:47:49.613407+0000 mon.a (mon.0) 667 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:47:50.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:50 smithi134 bash[24068]: audit 2024-07-15T06:47:49.851344+0000 mon.a (mon.0) 668 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:47:50.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:50 smithi134 bash[24068]: audit 2024-07-15T06:47:49.862397+0000 mon.a (mon.0) 669 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:47:50.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:50 smithi134 bash[24068]: audit 2024-07-15T06:47:50.249354+0000 mon.a (mon.0) 670 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:47:50.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:50 smithi134 bash[24068]: audit 2024-07-15T06:47:50.257913+0000 mon.a (mon.0) 671 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:47:50.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:50 smithi134 bash[24068]: audit 2024-07-15T06:47:50.459498+0000 mon.a (mon.0) 672 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:47:50.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:50 smithi134 bash[24068]: audit 2024-07-15T06:47:50.468394+0000 mon.a (mon.0) 673 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:47:50.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:50 smithi134 bash[24068]: audit 2024-07-15T06:47:50.470725+0000 mon.a (mon.0) 674 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:47:50.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:50 smithi134 bash[24068]: audit 2024-07-15T06:47:50.472387+0000 mon.a (mon.0) 675 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:47:50.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:50 smithi134 bash[24068]: audit 2024-07-15T06:47:50.474156+0000 mon.a (mon.0) 676 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:47:50.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:50 smithi134 bash[24068]: audit 2024-07-15T06:47:50.475870+0000 mon.a (mon.0) 677 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:47:50.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:50 smithi134 bash[24068]: audit 2024-07-15T06:47:50.485062+0000 mon.a (mon.0) 678 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:47:50.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:50 smithi134 bash[24068]: audit 2024-07-15T06:47:50.487434+0000 mon.a (mon.0) 679 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:47:50.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:50 smithi134 bash[24068]: audit 2024-07-15T06:47:50.488988+0000 mon.a (mon.0) 680 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:47:50.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:50 smithi089 bash[21793]: audit 2024-07-15T06:47:49.607827+0000 mon.a (mon.0) 666 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:47:50.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:50 smithi089 bash[21793]: audit 2024-07-15T06:47:49.613407+0000 mon.a (mon.0) 667 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:47:50.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:50 smithi089 bash[21793]: audit 2024-07-15T06:47:49.851344+0000 mon.a (mon.0) 668 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:47:50.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:50 smithi089 bash[21793]: audit 2024-07-15T06:47:49.862397+0000 mon.a (mon.0) 669 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:47:50.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:50 smithi089 bash[21793]: audit 2024-07-15T06:47:50.249354+0000 mon.a (mon.0) 670 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:47:50.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:50 smithi089 bash[21793]: audit 2024-07-15T06:47:50.257913+0000 mon.a (mon.0) 671 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:47:50.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:50 smithi089 bash[21793]: audit 2024-07-15T06:47:50.459498+0000 mon.a (mon.0) 672 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:47:50.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:50 smithi089 bash[21793]: audit 2024-07-15T06:47:50.468394+0000 mon.a (mon.0) 673 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:47:50.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:50 smithi089 bash[21793]: audit 2024-07-15T06:47:50.470725+0000 mon.a (mon.0) 674 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:47:50.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:50 smithi089 bash[21793]: audit 2024-07-15T06:47:50.472387+0000 mon.a (mon.0) 675 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:47:50.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:50 smithi089 bash[21793]: audit 2024-07-15T06:47:50.474156+0000 mon.a (mon.0) 676 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:47:50.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:50 smithi089 bash[21793]: audit 2024-07-15T06:47:50.475870+0000 mon.a (mon.0) 677 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:47:50.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:50 smithi089 bash[21793]: audit 2024-07-15T06:47:50.485062+0000 mon.a (mon.0) 678 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:47:50.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:50 smithi089 bash[21793]: audit 2024-07-15T06:47:50.487434+0000 mon.a (mon.0) 679 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:47:50.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:50 smithi089 bash[21793]: audit 2024-07-15T06:47:50.488988+0000 mon.a (mon.0) 680 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:47:50.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:50 smithi089 bash[31372]: audit 2024-07-15T06:47:49.607827+0000 mon.a (mon.0) 666 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:47:50.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:50 smithi089 bash[31372]: audit 2024-07-15T06:47:49.613407+0000 mon.a (mon.0) 667 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:47:50.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:50 smithi089 bash[31372]: audit 2024-07-15T06:47:49.851344+0000 mon.a (mon.0) 668 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:47:50.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:50 smithi089 bash[31372]: audit 2024-07-15T06:47:49.862397+0000 mon.a (mon.0) 669 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:47:50.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:50 smithi089 bash[31372]: audit 2024-07-15T06:47:50.249354+0000 mon.a (mon.0) 670 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:47:50.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:50 smithi089 bash[31372]: audit 2024-07-15T06:47:50.257913+0000 mon.a (mon.0) 671 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:47:50.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:50 smithi089 bash[31372]: audit 2024-07-15T06:47:50.459498+0000 mon.a (mon.0) 672 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:47:50.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:50 smithi089 bash[31372]: audit 2024-07-15T06:47:50.468394+0000 mon.a (mon.0) 673 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:47:50.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:50 smithi089 bash[31372]: audit 2024-07-15T06:47:50.470725+0000 mon.a (mon.0) 674 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:47:50.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:50 smithi089 bash[31372]: audit 2024-07-15T06:47:50.472387+0000 mon.a (mon.0) 675 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:47:50.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:50 smithi089 bash[31372]: audit 2024-07-15T06:47:50.474156+0000 mon.a (mon.0) 676 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:47:50.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:50 smithi089 bash[31372]: audit 2024-07-15T06:47:50.475870+0000 mon.a (mon.0) 677 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:47:50.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:50 smithi089 bash[31372]: audit 2024-07-15T06:47:50.485062+0000 mon.a (mon.0) 678 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:47:50.873 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:50 smithi089 bash[31372]: audit 2024-07-15T06:47:50.487434+0000 mon.a (mon.0) 679 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:47:50.873 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:50 smithi089 bash[31372]: audit 2024-07-15T06:47:50.488988+0000 mon.a (mon.0) 680 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:47:51.553 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:51 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:47:51.554 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:51 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:47:51.554 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:47:51 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:47:51.554 INFO:journalctl@ceph.osd.0.smithi089.stdout:Jul 15 06:47:51 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:47:51.555 INFO:journalctl@ceph.osd.1.smithi089.stdout:Jul 15 06:47:51 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:47:51.555 INFO:journalctl@ceph.osd.3.smithi089.stdout:Jul 15 06:47:51 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:47:51.556 INFO:journalctl@ceph.osd.2.smithi089.stdout:Jul 15 06:47:51 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:47:51.556 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:47:51 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:47:51.802 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:47:51 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:47:51.802 INFO:journalctl@ceph.osd.0.smithi089.stdout:Jul 15 06:47:51 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:47:51.802 INFO:journalctl@ceph.osd.1.smithi089.stdout:Jul 15 06:47:51 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:47:51.803 INFO:journalctl@ceph.osd.3.smithi089.stdout:Jul 15 06:47:51 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:47:51.803 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:51 smithi089 bash[31372]: cluster 2024-07-15T06:47:50.104690+0000 mgr.y (mgr.24272) 10 : cluster [DBG] pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:47:51.803 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:51 smithi089 bash[31372]: cephadm 2024-07-15T06:47:50.477209+0000 mgr.y (mgr.24272) 11 : cephadm [INF] Adjusting osd_memory_target on smithi134 to 4060M 2024-07-15T06:47:51.803 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:51 smithi089 bash[31372]: cephadm 2024-07-15T06:47:50.491081+0000 mgr.y (mgr.24272) 12 : cephadm [INF] Updating smithi089:/etc/ceph/ceph.conf 2024-07-15T06:47:51.803 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:51 smithi089 bash[31372]: cephadm 2024-07-15T06:47:50.491731+0000 mgr.y (mgr.24272) 13 : cephadm [INF] Updating smithi134:/etc/ceph/ceph.conf 2024-07-15T06:47:51.803 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:51 smithi089 bash[31372]: cephadm 2024-07-15T06:47:50.667311+0000 mgr.y (mgr.24272) 14 : cephadm [INF] Updating smithi089:/var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/config/ceph.conf 2024-07-15T06:47:51.803 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:51 smithi089 bash[31372]: cephadm 2024-07-15T06:47:50.667472+0000 mgr.y (mgr.24272) 15 : cephadm [INF] Updating smithi134:/var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/config/ceph.conf 2024-07-15T06:47:51.803 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:51 smithi089 bash[31372]: audit 2024-07-15T06:47:51.024427+0000 mon.a (mon.0) 681 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:47:51.803 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:51 smithi089 bash[31372]: audit 2024-07-15T06:47:51.032485+0000 mon.a (mon.0) 682 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:47:51.803 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:51 smithi089 bash[31372]: audit 2024-07-15T06:47:51.064489+0000 mon.a (mon.0) 683 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:47:51.804 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:51 smithi089 bash[31372]: audit 2024-07-15T06:47:51.073525+0000 mon.a (mon.0) 684 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:47:51.804 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:51 smithi089 bash[31372]: audit 2024-07-15T06:47:51.083344+0000 mon.a (mon.0) 685 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:47:51.804 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:51 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:47:51.805 INFO:journalctl@ceph.osd.2.smithi089.stdout:Jul 15 06:47:51 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:47:51.805 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:51 smithi089 bash[21793]: cluster 2024-07-15T06:47:50.104690+0000 mgr.y (mgr.24272) 10 : cluster [DBG] pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:47:51.805 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:51 smithi089 bash[21793]: cephadm 2024-07-15T06:47:50.477209+0000 mgr.y (mgr.24272) 11 : cephadm [INF] Adjusting osd_memory_target on smithi134 to 4060M 2024-07-15T06:47:51.805 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:51 smithi089 bash[21793]: cephadm 2024-07-15T06:47:50.491081+0000 mgr.y (mgr.24272) 12 : cephadm [INF] Updating smithi089:/etc/ceph/ceph.conf 2024-07-15T06:47:51.806 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:51 smithi089 bash[21793]: cephadm 2024-07-15T06:47:50.491731+0000 mgr.y (mgr.24272) 13 : cephadm [INF] Updating smithi134:/etc/ceph/ceph.conf 2024-07-15T06:47:51.806 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:51 smithi089 bash[21793]: cephadm 2024-07-15T06:47:50.667311+0000 mgr.y (mgr.24272) 14 : cephadm [INF] Updating smithi089:/var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/config/ceph.conf 2024-07-15T06:47:51.806 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:51 smithi089 bash[21793]: cephadm 2024-07-15T06:47:50.667472+0000 mgr.y (mgr.24272) 15 : cephadm [INF] Updating smithi134:/var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/config/ceph.conf 2024-07-15T06:47:51.806 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:51 smithi089 bash[21793]: audit 2024-07-15T06:47:51.024427+0000 mon.a (mon.0) 681 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:47:51.806 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:51 smithi089 bash[21793]: audit 2024-07-15T06:47:51.032485+0000 mon.a (mon.0) 682 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:47:51.806 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:51 smithi089 bash[21793]: audit 2024-07-15T06:47:51.064489+0000 mon.a (mon.0) 683 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:47:51.806 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:51 smithi089 bash[21793]: audit 2024-07-15T06:47:51.073525+0000 mon.a (mon.0) 684 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:47:51.806 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:51 smithi089 bash[21793]: audit 2024-07-15T06:47:51.083344+0000 mon.a (mon.0) 685 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:47:51.806 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:51 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:47:51.807 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:47:51 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:47:51.807 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:47:51 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:47:51.807 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:47:51 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:47:51.807 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:47:51 smithi089 systemd[1]: Started Ceph node-exporter.a for 75ba02bc-4274-11ef-bcad-c7b262605968. 2024-07-15T06:47:52.017 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:51 smithi134 bash[24068]: cluster 2024-07-15T06:47:50.104690+0000 mgr.y (mgr.24272) 10 : cluster [DBG] pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:47:52.017 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:51 smithi134 bash[24068]: cephadm 2024-07-15T06:47:50.477209+0000 mgr.y (mgr.24272) 11 : cephadm [INF] Adjusting osd_memory_target on smithi134 to 4060M 2024-07-15T06:47:52.017 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:51 smithi134 bash[24068]: cephadm 2024-07-15T06:47:50.491081+0000 mgr.y (mgr.24272) 12 : cephadm [INF] Updating smithi089:/etc/ceph/ceph.conf 2024-07-15T06:47:52.018 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:51 smithi134 bash[24068]: cephadm 2024-07-15T06:47:50.491731+0000 mgr.y (mgr.24272) 13 : cephadm [INF] Updating smithi134:/etc/ceph/ceph.conf 2024-07-15T06:47:52.018 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:51 smithi134 bash[24068]: cephadm 2024-07-15T06:47:50.667311+0000 mgr.y (mgr.24272) 14 : cephadm [INF] Updating smithi089:/var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/config/ceph.conf 2024-07-15T06:47:52.018 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:51 smithi134 bash[24068]: cephadm 2024-07-15T06:47:50.667472+0000 mgr.y (mgr.24272) 15 : cephadm [INF] Updating smithi134:/var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/config/ceph.conf 2024-07-15T06:47:52.018 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:51 smithi134 bash[24068]: audit 2024-07-15T06:47:51.024427+0000 mon.a (mon.0) 681 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:47:52.018 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:51 smithi134 bash[24068]: audit 2024-07-15T06:47:51.032485+0000 mon.a (mon.0) 682 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:47:52.018 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:51 smithi134 bash[24068]: audit 2024-07-15T06:47:51.064489+0000 mon.a (mon.0) 683 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:47:52.018 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:51 smithi134 bash[24068]: audit 2024-07-15T06:47:51.073525+0000 mon.a (mon.0) 684 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:47:52.018 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:51 smithi134 bash[24068]: audit 2024-07-15T06:47:51.083344+0000 mon.a (mon.0) 685 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:47:52.119 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:47:51 smithi089 bash[55935]: Unable to find image 'quay.io/prometheus/node-exporter:v1.5.0' locally 2024-07-15T06:47:52.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:52 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:47:52.306 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:47:52 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:47:52.306 INFO:journalctl@ceph.osd.4.smithi134.stdout:Jul 15 06:47:52 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:47:52.306 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 06:47:52 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:47:52.307 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:47:52 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:47:52.307 INFO:journalctl@ceph.osd.6.smithi134.stdout:Jul 15 06:47:52 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:47:52.307 INFO:journalctl@ceph.osd.5.smithi134.stdout:Jul 15 06:47:52 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:47:52.574 INFO:journalctl@ceph.osd.4.smithi134.stdout:Jul 15 06:47:52 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:47:52.574 INFO:journalctl@ceph.osd.5.smithi134.stdout:Jul 15 06:47:52 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:47:52.575 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:47:52 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:47:52.575 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:52 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:47:52.575 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 06:47:52 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:47:52.575 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:47:52 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:47:52.576 INFO:journalctl@ceph.osd.6.smithi134.stdout:Jul 15 06:47:52 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:47:52.619 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:47:52 smithi089 bash[55935]: v1.5.0: Pulling from prometheus/node-exporter 2024-07-15T06:47:52.620 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:47:52 smithi089 bash[55935]: 22b70bddd3ac: Pulling fs layer 2024-07-15T06:47:52.620 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:47:52 smithi089 bash[55935]: 5c12815fee55: Pulling fs layer 2024-07-15T06:47:52.620 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:47:52 smithi089 bash[55935]: c0e87333d380: Pulling fs layer 2024-07-15T06:47:52.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:52 smithi089 bash[21793]: cephadm 2024-07-15T06:47:50.781579+0000 mgr.y (mgr.24272) 16 : cephadm [INF] Updating smithi089:/etc/ceph/ceph.client.admin.keyring 2024-07-15T06:47:52.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:52 smithi089 bash[21793]: cephadm 2024-07-15T06:47:50.803111+0000 mgr.y (mgr.24272) 17 : cephadm [INF] Updating smithi134:/etc/ceph/ceph.client.admin.keyring 2024-07-15T06:47:52.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:52 smithi089 bash[21793]: cephadm 2024-07-15T06:47:50.902147+0000 mgr.y (mgr.24272) 18 : cephadm [INF] Updating smithi089:/var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/config/ceph.client.admin.keyring 2024-07-15T06:47:52.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:52 smithi089 bash[21793]: cephadm 2024-07-15T06:47:50.935815+0000 mgr.y (mgr.24272) 19 : cephadm [INF] Updating smithi134:/var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/config/ceph.client.admin.keyring 2024-07-15T06:47:52.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:52 smithi089 bash[21793]: cephadm 2024-07-15T06:47:51.086754+0000 mgr.y (mgr.24272) 20 : cephadm [INF] Deploying daemon node-exporter.a on smithi089 2024-07-15T06:47:52.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:52 smithi089 bash[21793]: audit 2024-07-15T06:47:51.839014+0000 mon.a (mon.0) 686 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:47:52.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:52 smithi089 bash[21793]: audit 2024-07-15T06:47:51.844858+0000 mon.a (mon.0) 687 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:47:52.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:52 smithi089 bash[21793]: audit 2024-07-15T06:47:51.849691+0000 mon.a (mon.0) 688 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:47:52.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:52 smithi089 bash[31372]: cephadm 2024-07-15T06:47:50.781579+0000 mgr.y (mgr.24272) 16 : cephadm [INF] Updating smithi089:/etc/ceph/ceph.client.admin.keyring 2024-07-15T06:47:52.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:52 smithi089 bash[31372]: cephadm 2024-07-15T06:47:50.803111+0000 mgr.y (mgr.24272) 17 : cephadm [INF] Updating smithi134:/etc/ceph/ceph.client.admin.keyring 2024-07-15T06:47:52.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:52 smithi089 bash[31372]: cephadm 2024-07-15T06:47:50.902147+0000 mgr.y (mgr.24272) 18 : cephadm [INF] Updating smithi089:/var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/config/ceph.client.admin.keyring 2024-07-15T06:47:52.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:52 smithi089 bash[31372]: cephadm 2024-07-15T06:47:50.935815+0000 mgr.y (mgr.24272) 19 : cephadm [INF] Updating smithi134:/var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/config/ceph.client.admin.keyring 2024-07-15T06:47:52.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:52 smithi089 bash[31372]: cephadm 2024-07-15T06:47:51.086754+0000 mgr.y (mgr.24272) 20 : cephadm [INF] Deploying daemon node-exporter.a on smithi089 2024-07-15T06:47:52.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:52 smithi089 bash[31372]: audit 2024-07-15T06:47:51.839014+0000 mon.a (mon.0) 686 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:47:52.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:52 smithi089 bash[31372]: audit 2024-07-15T06:47:51.844858+0000 mon.a (mon.0) 687 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:47:52.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:52 smithi089 bash[31372]: audit 2024-07-15T06:47:51.849691+0000 mon.a (mon.0) 688 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:47:53.055 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:47:52 smithi134 systemd[1]: Started Ceph node-exporter.b for 75ba02bc-4274-11ef-bcad-c7b262605968. 2024-07-15T06:47:53.055 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:47:52 smithi134 bash[49200]: Unable to find image 'quay.io/prometheus/node-exporter:v1.5.0' locally 2024-07-15T06:47:53.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:52 smithi134 bash[24068]: cephadm 2024-07-15T06:47:50.781579+0000 mgr.y (mgr.24272) 16 : cephadm [INF] Updating smithi089:/etc/ceph/ceph.client.admin.keyring 2024-07-15T06:47:53.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:52 smithi134 bash[24068]: cephadm 2024-07-15T06:47:50.803111+0000 mgr.y (mgr.24272) 17 : cephadm [INF] Updating smithi134:/etc/ceph/ceph.client.admin.keyring 2024-07-15T06:47:53.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:52 smithi134 bash[24068]: cephadm 2024-07-15T06:47:50.902147+0000 mgr.y (mgr.24272) 18 : cephadm [INF] Updating smithi089:/var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/config/ceph.client.admin.keyring 2024-07-15T06:47:53.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:52 smithi134 bash[24068]: cephadm 2024-07-15T06:47:50.935815+0000 mgr.y (mgr.24272) 19 : cephadm [INF] Updating smithi134:/var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/config/ceph.client.admin.keyring 2024-07-15T06:47:53.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:52 smithi134 bash[24068]: cephadm 2024-07-15T06:47:51.086754+0000 mgr.y (mgr.24272) 20 : cephadm [INF] Deploying daemon node-exporter.a on smithi089 2024-07-15T06:47:53.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:52 smithi134 bash[24068]: audit 2024-07-15T06:47:51.839014+0000 mon.a (mon.0) 686 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:47:53.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:52 smithi134 bash[24068]: audit 2024-07-15T06:47:51.844858+0000 mon.a (mon.0) 687 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:47:53.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:52 smithi134 bash[24068]: audit 2024-07-15T06:47:51.849691+0000 mon.a (mon.0) 688 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:47:53.119 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:47:52 smithi089 bash[55935]: 5c12815fee55: Download complete 2024-07-15T06:47:53.120 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:47:52 smithi089 bash[55935]: 22b70bddd3ac: Verifying Checksum 2024-07-15T06:47:53.120 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:47:52 smithi089 bash[55935]: 22b70bddd3ac: Download complete 2024-07-15T06:47:53.120 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:47:52 smithi089 bash[55935]: c0e87333d380: Verifying Checksum 2024-07-15T06:47:53.120 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:47:52 smithi089 bash[55935]: c0e87333d380: Download complete 2024-07-15T06:47:53.120 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:47:52 smithi089 bash[55935]: 22b70bddd3ac: Pull complete 2024-07-15T06:47:53.120 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 06:47:52 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:47:53.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:53 smithi089 bash[21793]: cephadm 2024-07-15T06:47:51.850299+0000 mgr.y (mgr.24272) 21 : cephadm [INF] Deploying daemon node-exporter.b on smithi134 2024-07-15T06:47:53.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:53 smithi089 bash[21793]: cluster 2024-07-15T06:47:52.105262+0000 mgr.y (mgr.24272) 22 : cluster [DBG] pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 16 op/s 2024-07-15T06:47:53.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:53 smithi089 bash[21793]: audit 2024-07-15T06:47:52.624091+0000 mon.a (mon.0) 689 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:47:53.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:53 smithi089 bash[21793]: audit 2024-07-15T06:47:52.635263+0000 mon.a (mon.0) 690 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:47:53.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:53 smithi089 bash[21793]: audit 2024-07-15T06:47:52.645365+0000 mon.a (mon.0) 691 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:47:53.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:53 smithi089 bash[21793]: audit 2024-07-15T06:47:52.655531+0000 mon.a (mon.0) 692 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:47:53.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:53 smithi089 bash[21793]: audit 2024-07-15T06:47:52.670165+0000 mon.a (mon.0) 693 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:47:53.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:53 smithi089 bash[21793]: cephadm 2024-07-15T06:47:52.697587+0000 mgr.y (mgr.24272) 23 : cephadm [INF] Deploying daemon alertmanager.a on smithi089 2024-07-15T06:47:53.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:53 smithi089 bash[31372]: cephadm 2024-07-15T06:47:51.850299+0000 mgr.y (mgr.24272) 21 : cephadm [INF] Deploying daemon node-exporter.b on smithi134 2024-07-15T06:47:53.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:53 smithi089 bash[31372]: cluster 2024-07-15T06:47:52.105262+0000 mgr.y (mgr.24272) 22 : cluster [DBG] pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 16 op/s 2024-07-15T06:47:53.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:53 smithi089 bash[31372]: audit 2024-07-15T06:47:52.624091+0000 mon.a (mon.0) 689 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:47:53.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:53 smithi089 bash[31372]: audit 2024-07-15T06:47:52.635263+0000 mon.a (mon.0) 690 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:47:53.873 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:53 smithi089 bash[31372]: audit 2024-07-15T06:47:52.645365+0000 mon.a (mon.0) 691 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:47:53.873 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:53 smithi089 bash[31372]: audit 2024-07-15T06:47:52.655531+0000 mon.a (mon.0) 692 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:47:53.873 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:53 smithi089 bash[31372]: audit 2024-07-15T06:47:52.670165+0000 mon.a (mon.0) 693 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:47:53.873 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:53 smithi089 bash[31372]: cephadm 2024-07-15T06:47:52.697587+0000 mgr.y (mgr.24272) 23 : cephadm [INF] Deploying daemon alertmanager.a on smithi089 2024-07-15T06:47:53.984 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:53 smithi134 bash[24068]: cephadm 2024-07-15T06:47:51.850299+0000 mgr.y (mgr.24272) 21 : cephadm [INF] Deploying daemon node-exporter.b on smithi134 2024-07-15T06:47:53.984 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:53 smithi134 bash[24068]: cluster 2024-07-15T06:47:52.105262+0000 mgr.y (mgr.24272) 22 : cluster [DBG] pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 16 op/s 2024-07-15T06:47:53.984 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:53 smithi134 bash[24068]: audit 2024-07-15T06:47:52.624091+0000 mon.a (mon.0) 689 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:47:53.984 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:53 smithi134 bash[24068]: audit 2024-07-15T06:47:52.635263+0000 mon.a (mon.0) 690 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:47:53.984 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:53 smithi134 bash[24068]: audit 2024-07-15T06:47:52.645365+0000 mon.a (mon.0) 691 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:47:53.984 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:53 smithi134 bash[24068]: audit 2024-07-15T06:47:52.655531+0000 mon.a (mon.0) 692 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:47:53.984 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:53 smithi134 bash[24068]: audit 2024-07-15T06:47:52.670165+0000 mon.a (mon.0) 693 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:47:53.984 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:53 smithi134 bash[24068]: cephadm 2024-07-15T06:47:52.697587+0000 mgr.y (mgr.24272) 23 : cephadm [INF] Deploying daemon alertmanager.a on smithi089 2024-07-15T06:47:53.985 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:47:53 smithi134 bash[49200]: v1.5.0: Pulling from prometheus/node-exporter 2024-07-15T06:47:53.985 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:47:53 smithi134 bash[49200]: 22b70bddd3ac: Pulling fs layer 2024-07-15T06:47:53.985 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:47:53 smithi134 bash[49200]: 5c12815fee55: Pulling fs layer 2024-07-15T06:47:53.985 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:47:53 smithi134 bash[49200]: c0e87333d380: Pulling fs layer 2024-07-15T06:47:54.179 INFO:teuthology.orchestra.run.smithi134.stdout:Scheduled grafana update... 2024-07-15T06:47:54.305 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:47:53 smithi134 bash[49200]: 22b70bddd3ac: Download complete 2024-07-15T06:47:54.305 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:47:54 smithi134 bash[49200]: 5c12815fee55: Download complete 2024-07-15T06:47:54.305 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:47:54 smithi134 bash[49200]: c0e87333d380: Verifying Checksum 2024-07-15T06:47:54.305 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:47:54 smithi134 bash[49200]: c0e87333d380: Download complete 2024-07-15T06:47:54.305 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:47:54 smithi134 bash[49200]: 22b70bddd3ac: Pull complete 2024-07-15T06:47:54.369 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:47:53 smithi089 bash[55935]: 5c12815fee55: Pull complete 2024-07-15T06:47:54.619 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:47:54 smithi089 bash[55935]: c0e87333d380: Pull complete 2024-07-15T06:47:55.120 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:47:54 smithi089 bash[55935]: Digest: sha256:39c642b2b337e38c18e80266fb14383754178202f40103646337722a594d984c 2024-07-15T06:47:55.120 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:47:54 smithi089 bash[55935]: Status: Downloaded newer image for quay.io/prometheus/node-exporter:v1.5.0 2024-07-15T06:47:55.320 DEBUG:teuthology.orchestra.run.smithi134:grafana.a> sudo journalctl -f -n 0 -u ceph-75ba02bc-4274-11ef-bcad-c7b262605968@grafana.a.service 2024-07-15T06:47:55.322 INFO:tasks.cephadm:Setting up client nodes... 2024-07-15T06:47:55.323 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-07-15T06:47:55.555 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:47:55 smithi134 bash[49200]: 5c12815fee55: Pull complete 2024-07-15T06:47:55.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:55 smithi134 bash[24068]: cluster 2024-07-15T06:47:54.105759+0000 mgr.y (mgr.24272) 24 : cluster [DBG] pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-07-15T06:47:55.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:55 smithi134 bash[24068]: audit 2024-07-15T06:47:54.165375+0000 mgr.y (mgr.24272) 25 : audit [DBG] from='client.24343 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi134=a", "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T06:47:55.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:55 smithi134 bash[24068]: cephadm 2024-07-15T06:47:54.167494+0000 mgr.y (mgr.24272) 26 : cephadm [INF] Saving service grafana spec with placement smithi134=a;count:1 2024-07-15T06:47:55.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:55 smithi134 bash[24068]: audit 2024-07-15T06:47:54.172380+0000 mon.a (mon.0) 694 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:47:55.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:55 smithi134 bash[24068]: audit 2024-07-15T06:47:54.179133+0000 mon.a (mon.0) 695 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:47:55.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:55 smithi089 bash[21793]: cluster 2024-07-15T06:47:54.105759+0000 mgr.y (mgr.24272) 24 : cluster [DBG] pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-07-15T06:47:55.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:55 smithi089 bash[21793]: audit 2024-07-15T06:47:54.165375+0000 mgr.y (mgr.24272) 25 : audit [DBG] from='client.24343 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi134=a", "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T06:47:55.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:55 smithi089 bash[21793]: cephadm 2024-07-15T06:47:54.167494+0000 mgr.y (mgr.24272) 26 : cephadm [INF] Saving service grafana spec with placement smithi134=a;count:1 2024-07-15T06:47:55.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:55 smithi089 bash[21793]: audit 2024-07-15T06:47:54.172380+0000 mon.a (mon.0) 694 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:47:55.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:55 smithi089 bash[21793]: audit 2024-07-15T06:47:54.179133+0000 mon.a (mon.0) 695 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:47:55.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:55 smithi089 bash[31372]: cluster 2024-07-15T06:47:54.105759+0000 mgr.y (mgr.24272) 24 : cluster [DBG] pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-07-15T06:47:55.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:55 smithi089 bash[31372]: audit 2024-07-15T06:47:54.165375+0000 mgr.y (mgr.24272) 25 : audit [DBG] from='client.24343 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi134=a", "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T06:47:55.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:55 smithi089 bash[31372]: cephadm 2024-07-15T06:47:54.167494+0000 mgr.y (mgr.24272) 26 : cephadm [INF] Saving service grafana spec with placement smithi134=a;count:1 2024-07-15T06:47:55.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:55 smithi089 bash[31372]: audit 2024-07-15T06:47:54.172380+0000 mon.a (mon.0) 694 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:47:55.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:55 smithi089 bash[31372]: audit 2024-07-15T06:47:54.179133+0000 mon.a (mon.0) 695 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:47:56.046 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:47:55 smithi134 bash[49200]: c0e87333d380: Pull complete 2024-07-15T06:47:56.046 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:47:55 smithi134 bash[49200]: Digest: sha256:39c642b2b337e38c18e80266fb14383754178202f40103646337722a594d984c 2024-07-15T06:47:56.046 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:47:55 smithi134 bash[49200]: Status: Downloaded newer image for quay.io/prometheus/node-exporter:v1.5.0 2024-07-15T06:47:57.251 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:47:56 smithi134 bash[25019]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:47:56] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-07-15T06:47:57.252 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:57 smithi134 bash[24068]: cluster 2024-07-15T06:47:56.106358+0000 mgr.y (mgr.24272) 27 : cluster [DBG] pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-07-15T06:47:57.556 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:47:57 smithi134 bash[49200]: ts=2024-07-15T06:47:57.256Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-07-15T06:47:57.556 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:47:57 smithi134 bash[49200]: ts=2024-07-15T06:47:57.256Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-07-15T06:47:57.556 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:47:57 smithi134 bash[49200]: ts=2024-07-15T06:47:57.257Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-07-15T06:47:57.556 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:47:57 smithi134 bash[49200]: ts=2024-07-15T06:47:57.257Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-07-15T06:47:57.556 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:47:57 smithi134 bash[49200]: ts=2024-07-15T06:47:57.257Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-07-15T06:47:57.556 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:47:57 smithi134 bash[49200]: ts=2024-07-15T06:47:57.257Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-07-15T06:47:57.556 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:47:57 smithi134 bash[49200]: ts=2024-07-15T06:47:57.258Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-07-15T06:47:57.557 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:47:57 smithi134 bash[49200]: ts=2024-07-15T06:47:57.258Z caller=node_exporter.go:117 level=info collector=arp 2024-07-15T06:47:57.557 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:47:57 smithi134 bash[49200]: ts=2024-07-15T06:47:57.258Z caller=node_exporter.go:117 level=info collector=bcache 2024-07-15T06:47:57.557 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:47:57 smithi134 bash[49200]: ts=2024-07-15T06:47:57.258Z caller=node_exporter.go:117 level=info collector=bonding 2024-07-15T06:47:57.557 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:47:57 smithi134 bash[49200]: ts=2024-07-15T06:47:57.258Z caller=node_exporter.go:117 level=info collector=btrfs 2024-07-15T06:47:57.557 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:47:57 smithi134 bash[49200]: ts=2024-07-15T06:47:57.258Z caller=node_exporter.go:117 level=info collector=conntrack 2024-07-15T06:47:57.557 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:47:57 smithi134 bash[49200]: ts=2024-07-15T06:47:57.258Z caller=node_exporter.go:117 level=info collector=cpu 2024-07-15T06:47:57.557 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:47:57 smithi134 bash[49200]: ts=2024-07-15T06:47:57.258Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-07-15T06:47:57.557 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:47:57 smithi134 bash[49200]: ts=2024-07-15T06:47:57.258Z caller=node_exporter.go:117 level=info collector=diskstats 2024-07-15T06:47:57.557 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:47:57 smithi134 bash[49200]: ts=2024-07-15T06:47:57.258Z caller=node_exporter.go:117 level=info collector=dmi 2024-07-15T06:47:57.557 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:47:57 smithi134 bash[49200]: ts=2024-07-15T06:47:57.258Z caller=node_exporter.go:117 level=info collector=edac 2024-07-15T06:47:57.557 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:47:57 smithi134 bash[49200]: ts=2024-07-15T06:47:57.258Z caller=node_exporter.go:117 level=info collector=entropy 2024-07-15T06:47:57.557 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:47:57 smithi134 bash[49200]: ts=2024-07-15T06:47:57.258Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-07-15T06:47:57.558 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:47:57 smithi134 bash[49200]: ts=2024-07-15T06:47:57.258Z caller=node_exporter.go:117 level=info collector=filefd 2024-07-15T06:47:57.558 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:47:57 smithi134 bash[49200]: ts=2024-07-15T06:47:57.258Z caller=node_exporter.go:117 level=info collector=filesystem 2024-07-15T06:47:57.558 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:47:57 smithi134 bash[49200]: ts=2024-07-15T06:47:57.258Z caller=node_exporter.go:117 level=info collector=hwmon 2024-07-15T06:47:57.558 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:47:57 smithi134 bash[49200]: ts=2024-07-15T06:47:57.258Z caller=node_exporter.go:117 level=info collector=infiniband 2024-07-15T06:47:57.558 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:47:57 smithi134 bash[49200]: ts=2024-07-15T06:47:57.258Z caller=node_exporter.go:117 level=info collector=ipvs 2024-07-15T06:47:57.558 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:47:57 smithi134 bash[49200]: ts=2024-07-15T06:47:57.258Z caller=node_exporter.go:117 level=info collector=loadavg 2024-07-15T06:47:57.558 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:47:57 smithi134 bash[49200]: ts=2024-07-15T06:47:57.258Z caller=node_exporter.go:117 level=info collector=mdadm 2024-07-15T06:47:57.558 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:47:57 smithi134 bash[49200]: ts=2024-07-15T06:47:57.258Z caller=node_exporter.go:117 level=info collector=meminfo 2024-07-15T06:47:57.559 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:47:57 smithi134 bash[49200]: ts=2024-07-15T06:47:57.258Z caller=node_exporter.go:117 level=info collector=netclass 2024-07-15T06:47:57.559 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:47:57 smithi134 bash[49200]: ts=2024-07-15T06:47:57.258Z caller=node_exporter.go:117 level=info collector=netdev 2024-07-15T06:47:57.559 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:47:57 smithi134 bash[49200]: ts=2024-07-15T06:47:57.258Z caller=node_exporter.go:117 level=info collector=netstat 2024-07-15T06:47:57.559 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:47:57 smithi134 bash[49200]: ts=2024-07-15T06:47:57.258Z caller=node_exporter.go:117 level=info collector=nfs 2024-07-15T06:47:57.559 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:47:57 smithi134 bash[49200]: ts=2024-07-15T06:47:57.258Z caller=node_exporter.go:117 level=info collector=nfsd 2024-07-15T06:47:57.559 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:47:57 smithi134 bash[49200]: ts=2024-07-15T06:47:57.258Z caller=node_exporter.go:117 level=info collector=nvme 2024-07-15T06:47:57.559 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:47:57 smithi134 bash[49200]: ts=2024-07-15T06:47:57.258Z caller=node_exporter.go:117 level=info collector=os 2024-07-15T06:47:57.559 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:47:57 smithi134 bash[49200]: ts=2024-07-15T06:47:57.258Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-07-15T06:47:57.559 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:47:57 smithi134 bash[49200]: ts=2024-07-15T06:47:57.258Z caller=node_exporter.go:117 level=info collector=pressure 2024-07-15T06:47:57.559 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:47:57 smithi134 bash[49200]: ts=2024-07-15T06:47:57.258Z caller=node_exporter.go:117 level=info collector=rapl 2024-07-15T06:47:57.559 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:47:57 smithi134 bash[49200]: ts=2024-07-15T06:47:57.258Z caller=node_exporter.go:117 level=info collector=schedstat 2024-07-15T06:47:57.559 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:47:57 smithi134 bash[49200]: ts=2024-07-15T06:47:57.258Z caller=node_exporter.go:117 level=info collector=selinux 2024-07-15T06:47:57.559 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:47:57 smithi134 bash[49200]: ts=2024-07-15T06:47:57.258Z caller=node_exporter.go:117 level=info collector=sockstat 2024-07-15T06:47:57.560 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:47:57 smithi134 bash[49200]: ts=2024-07-15T06:47:57.258Z caller=node_exporter.go:117 level=info collector=softnet 2024-07-15T06:47:57.560 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:47:57 smithi134 bash[49200]: ts=2024-07-15T06:47:57.258Z caller=node_exporter.go:117 level=info collector=stat 2024-07-15T06:47:57.560 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:47:57 smithi134 bash[49200]: ts=2024-07-15T06:47:57.258Z caller=node_exporter.go:117 level=info collector=tapestats 2024-07-15T06:47:57.560 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:47:57 smithi134 bash[49200]: ts=2024-07-15T06:47:57.258Z caller=node_exporter.go:117 level=info collector=textfile 2024-07-15T06:47:57.560 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:47:57 smithi134 bash[49200]: ts=2024-07-15T06:47:57.258Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-07-15T06:47:57.560 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:47:57 smithi134 bash[49200]: ts=2024-07-15T06:47:57.258Z caller=node_exporter.go:117 level=info collector=time 2024-07-15T06:47:57.560 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:47:57 smithi134 bash[49200]: ts=2024-07-15T06:47:57.258Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-07-15T06:47:57.560 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:47:57 smithi134 bash[49200]: ts=2024-07-15T06:47:57.258Z caller=node_exporter.go:117 level=info collector=uname 2024-07-15T06:47:57.560 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:47:57 smithi134 bash[49200]: ts=2024-07-15T06:47:57.258Z caller=node_exporter.go:117 level=info collector=vmstat 2024-07-15T06:47:57.560 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:47:57 smithi134 bash[49200]: ts=2024-07-15T06:47:57.258Z caller=node_exporter.go:117 level=info collector=xfs 2024-07-15T06:47:57.560 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:47:57 smithi134 bash[49200]: ts=2024-07-15T06:47:57.258Z caller=node_exporter.go:117 level=info collector=zfs 2024-07-15T06:47:57.560 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:47:57 smithi134 bash[49200]: ts=2024-07-15T06:47:57.258Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-07-15T06:47:57.560 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:47:57 smithi134 bash[49200]: ts=2024-07-15T06:47:57.258Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-07-15T06:47:57.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:57 smithi089 bash[21793]: cluster 2024-07-15T06:47:56.106358+0000 mgr.y (mgr.24272) 27 : cluster [DBG] pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-07-15T06:47:57.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:57 smithi089 bash[31372]: cluster 2024-07-15T06:47:56.106358+0000 mgr.y (mgr.24272) 27 : cluster [DBG] pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-07-15T06:47:57.620 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:47:57 smithi089 bash[55935]: ts=2024-07-15T06:47:57.217Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-07-15T06:47:57.620 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:47:57 smithi089 bash[55935]: ts=2024-07-15T06:47:57.218Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-07-15T06:47:57.620 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:47:57 smithi089 bash[55935]: ts=2024-07-15T06:47:57.218Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-07-15T06:47:57.621 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:47:57 smithi089 bash[55935]: ts=2024-07-15T06:47:57.219Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-07-15T06:47:57.621 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:47:57 smithi089 bash[55935]: ts=2024-07-15T06:47:57.219Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-07-15T06:47:57.621 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:47:57 smithi089 bash[55935]: ts=2024-07-15T06:47:57.219Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-07-15T06:47:57.621 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:47:57 smithi089 bash[55935]: ts=2024-07-15T06:47:57.219Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-07-15T06:47:57.621 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:47:57 smithi089 bash[55935]: ts=2024-07-15T06:47:57.219Z caller=node_exporter.go:117 level=info collector=arp 2024-07-15T06:47:57.621 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:47:57 smithi089 bash[55935]: ts=2024-07-15T06:47:57.219Z caller=node_exporter.go:117 level=info collector=bcache 2024-07-15T06:47:57.621 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:47:57 smithi089 bash[55935]: ts=2024-07-15T06:47:57.219Z caller=node_exporter.go:117 level=info collector=bonding 2024-07-15T06:47:57.621 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:47:57 smithi089 bash[55935]: ts=2024-07-15T06:47:57.219Z caller=node_exporter.go:117 level=info collector=btrfs 2024-07-15T06:47:57.621 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:47:57 smithi089 bash[55935]: ts=2024-07-15T06:47:57.219Z caller=node_exporter.go:117 level=info collector=conntrack 2024-07-15T06:47:57.621 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:47:57 smithi089 bash[55935]: ts=2024-07-15T06:47:57.219Z caller=node_exporter.go:117 level=info collector=cpu 2024-07-15T06:47:57.621 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:47:57 smithi089 bash[55935]: ts=2024-07-15T06:47:57.219Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-07-15T06:47:57.621 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:47:57 smithi089 bash[55935]: ts=2024-07-15T06:47:57.219Z caller=node_exporter.go:117 level=info collector=diskstats 2024-07-15T06:47:57.622 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:47:57 smithi089 bash[55935]: ts=2024-07-15T06:47:57.219Z caller=node_exporter.go:117 level=info collector=dmi 2024-07-15T06:47:57.622 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:47:57 smithi089 bash[55935]: ts=2024-07-15T06:47:57.219Z caller=node_exporter.go:117 level=info collector=edac 2024-07-15T06:47:57.622 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:47:57 smithi089 bash[55935]: ts=2024-07-15T06:47:57.219Z caller=node_exporter.go:117 level=info collector=entropy 2024-07-15T06:47:57.622 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:47:57 smithi089 bash[55935]: ts=2024-07-15T06:47:57.219Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-07-15T06:47:57.622 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:47:57 smithi089 bash[55935]: ts=2024-07-15T06:47:57.219Z caller=node_exporter.go:117 level=info collector=filefd 2024-07-15T06:47:57.622 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:47:57 smithi089 bash[55935]: ts=2024-07-15T06:47:57.219Z caller=node_exporter.go:117 level=info collector=filesystem 2024-07-15T06:47:57.622 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:47:57 smithi089 bash[55935]: ts=2024-07-15T06:47:57.219Z caller=node_exporter.go:117 level=info collector=hwmon 2024-07-15T06:47:57.622 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:47:57 smithi089 bash[55935]: ts=2024-07-15T06:47:57.219Z caller=node_exporter.go:117 level=info collector=infiniband 2024-07-15T06:47:57.622 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:47:57 smithi089 bash[55935]: ts=2024-07-15T06:47:57.219Z caller=node_exporter.go:117 level=info collector=ipvs 2024-07-15T06:47:57.622 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:47:57 smithi089 bash[55935]: ts=2024-07-15T06:47:57.219Z caller=node_exporter.go:117 level=info collector=loadavg 2024-07-15T06:47:57.622 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:47:57 smithi089 bash[55935]: ts=2024-07-15T06:47:57.219Z caller=node_exporter.go:117 level=info collector=mdadm 2024-07-15T06:47:57.623 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:47:57 smithi089 bash[55935]: ts=2024-07-15T06:47:57.219Z caller=node_exporter.go:117 level=info collector=meminfo 2024-07-15T06:47:57.623 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:47:57 smithi089 bash[55935]: ts=2024-07-15T06:47:57.219Z caller=node_exporter.go:117 level=info collector=netclass 2024-07-15T06:47:57.623 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:47:57 smithi089 bash[55935]: ts=2024-07-15T06:47:57.219Z caller=node_exporter.go:117 level=info collector=netdev 2024-07-15T06:47:57.623 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:47:57 smithi089 bash[55935]: ts=2024-07-15T06:47:57.219Z caller=node_exporter.go:117 level=info collector=netstat 2024-07-15T06:47:57.623 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:47:57 smithi089 bash[55935]: ts=2024-07-15T06:47:57.219Z caller=node_exporter.go:117 level=info collector=nfs 2024-07-15T06:47:57.623 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:47:57 smithi089 bash[55935]: ts=2024-07-15T06:47:57.219Z caller=node_exporter.go:117 level=info collector=nfsd 2024-07-15T06:47:57.623 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:47:57 smithi089 bash[55935]: ts=2024-07-15T06:47:57.219Z caller=node_exporter.go:117 level=info collector=nvme 2024-07-15T06:47:57.623 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:47:57 smithi089 bash[55935]: ts=2024-07-15T06:47:57.219Z caller=node_exporter.go:117 level=info collector=os 2024-07-15T06:47:57.623 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:47:57 smithi089 bash[55935]: ts=2024-07-15T06:47:57.219Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-07-15T06:47:57.623 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:47:57 smithi089 bash[55935]: ts=2024-07-15T06:47:57.219Z caller=node_exporter.go:117 level=info collector=pressure 2024-07-15T06:47:57.623 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:47:57 smithi089 bash[55935]: ts=2024-07-15T06:47:57.219Z caller=node_exporter.go:117 level=info collector=rapl 2024-07-15T06:47:57.623 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:47:57 smithi089 bash[55935]: ts=2024-07-15T06:47:57.219Z caller=node_exporter.go:117 level=info collector=schedstat 2024-07-15T06:47:57.624 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:47:57 smithi089 bash[55935]: ts=2024-07-15T06:47:57.219Z caller=node_exporter.go:117 level=info collector=selinux 2024-07-15T06:47:57.624 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:47:57 smithi089 bash[55935]: ts=2024-07-15T06:47:57.219Z caller=node_exporter.go:117 level=info collector=sockstat 2024-07-15T06:47:57.624 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:47:57 smithi089 bash[55935]: ts=2024-07-15T06:47:57.219Z caller=node_exporter.go:117 level=info collector=softnet 2024-07-15T06:47:57.624 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:47:57 smithi089 bash[55935]: ts=2024-07-15T06:47:57.219Z caller=node_exporter.go:117 level=info collector=stat 2024-07-15T06:47:57.624 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:47:57 smithi089 bash[55935]: ts=2024-07-15T06:47:57.219Z caller=node_exporter.go:117 level=info collector=tapestats 2024-07-15T06:47:57.624 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:47:57 smithi089 bash[55935]: ts=2024-07-15T06:47:57.219Z caller=node_exporter.go:117 level=info collector=textfile 2024-07-15T06:47:57.624 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:47:57 smithi089 bash[55935]: ts=2024-07-15T06:47:57.219Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-07-15T06:47:57.624 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:47:57 smithi089 bash[55935]: ts=2024-07-15T06:47:57.219Z caller=node_exporter.go:117 level=info collector=time 2024-07-15T06:47:57.624 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:47:57 smithi089 bash[55935]: ts=2024-07-15T06:47:57.219Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-07-15T06:47:57.624 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:47:57 smithi089 bash[55935]: ts=2024-07-15T06:47:57.219Z caller=node_exporter.go:117 level=info collector=uname 2024-07-15T06:47:57.624 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:47:57 smithi089 bash[55935]: ts=2024-07-15T06:47:57.219Z caller=node_exporter.go:117 level=info collector=vmstat 2024-07-15T06:47:57.624 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:47:57 smithi089 bash[55935]: ts=2024-07-15T06:47:57.219Z caller=node_exporter.go:117 level=info collector=xfs 2024-07-15T06:47:57.624 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:47:57 smithi089 bash[55935]: ts=2024-07-15T06:47:57.219Z caller=node_exporter.go:117 level=info collector=zfs 2024-07-15T06:47:57.624 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:47:57 smithi089 bash[55935]: ts=2024-07-15T06:47:57.220Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-07-15T06:47:57.624 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:47:57 smithi089 bash[55935]: ts=2024-07-15T06:47:57.220Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-07-15T06:47:59.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:47:59 smithi134 bash[24068]: cluster 2024-07-15T06:47:58.106983+0000 mgr.y (mgr.24272) 28 : cluster [DBG] pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-07-15T06:47:59.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:47:59 smithi089 bash[21793]: cluster 2024-07-15T06:47:58.106983+0000 mgr.y (mgr.24272) 28 : cluster [DBG] pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-07-15T06:47:59.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:47:59 smithi089 bash[31372]: cluster 2024-07-15T06:47:58.106983+0000 mgr.y (mgr.24272) 28 : cluster [DBG] pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-07-15T06:48:00.491 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:00 smithi089 bash[21793]: audit 2024-07-15T06:47:59.187979+0000 mon.a (mon.0) 696 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:48:00.491 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:00 smithi089 bash[31372]: audit 2024-07-15T06:47:59.187979+0000 mon.a (mon.0) 696 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:48:00.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:00 smithi134 bash[24068]: audit 2024-07-15T06:47:59.187979+0000 mon.a (mon.0) 696 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:48:01.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:01 smithi134 bash[24068]: cluster 2024-07-15T06:48:00.107624+0000 mgr.y (mgr.24272) 29 : cluster [DBG] pgmap v11: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-07-15T06:48:01.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:01 smithi089 bash[21793]: cluster 2024-07-15T06:48:00.107624+0000 mgr.y (mgr.24272) 29 : cluster [DBG] pgmap v11: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-07-15T06:48:01.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:01 smithi089 bash[31372]: cluster 2024-07-15T06:48:00.107624+0000 mgr.y (mgr.24272) 29 : cluster [DBG] pgmap v11: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-07-15T06:48:01.666 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T06:48:03.472 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:03 smithi089 bash[31372]: cluster 2024-07-15T06:48:02.108285+0000 mgr.y (mgr.24272) 30 : cluster [DBG] pgmap v12: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-07-15T06:48:03.473 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:03 smithi089 bash[21793]: cluster 2024-07-15T06:48:02.108285+0000 mgr.y (mgr.24272) 30 : cluster [DBG] pgmap v12: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-07-15T06:48:03.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:03 smithi134 bash[24068]: cluster 2024-07-15T06:48:02.108285+0000 mgr.y (mgr.24272) 30 : cluster [DBG] pgmap v12: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-07-15T06:48:04.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:04 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:48:04.620 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:48:04 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:48:04.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:04 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:48:04.621 INFO:journalctl@ceph.osd.0.smithi089.stdout:Jul 15 06:48:04 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:48:04.621 INFO:journalctl@ceph.osd.1.smithi089.stdout:Jul 15 06:48:04 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:48:04.621 INFO:journalctl@ceph.osd.3.smithi089.stdout:Jul 15 06:48:04 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:48:04.622 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:48:04 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:48:04.622 INFO:journalctl@ceph.osd.2.smithi089.stdout:Jul 15 06:48:04 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:48:04.622 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 06:48:04 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:48:04.622 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 06:48:04 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:48:04.622 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 06:48:04 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:48:05.119 INFO:journalctl@ceph.osd.2.smithi089.stdout:Jul 15 06:48:04 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:48:05.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:04 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:48:05.120 INFO:journalctl@ceph.osd.0.smithi089.stdout:Jul 15 06:48:04 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:48:05.120 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:48:04 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:48:05.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:04 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:48:05.121 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:48:04 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:48:05.121 INFO:journalctl@ceph.osd.1.smithi089.stdout:Jul 15 06:48:04 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:48:05.121 INFO:journalctl@ceph.osd.3.smithi089.stdout:Jul 15 06:48:04 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:48:05.122 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 06:48:04 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:48:05.122 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 06:48:04 smithi089 systemd[1]: Started Ceph alertmanager.a for 75ba02bc-4274-11ef-bcad-c7b262605968. 2024-07-15T06:48:05.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:05 smithi134 bash[24068]: cluster 2024-07-15T06:48:04.108829+0000 mgr.y (mgr.24272) 31 : cluster [DBG] pgmap v13: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:48:05.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:05 smithi134 bash[24068]: audit 2024-07-15T06:48:04.929804+0000 mon.a (mon.0) 697 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:05.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:05 smithi134 bash[24068]: audit 2024-07-15T06:48:04.934513+0000 mon.a (mon.0) 698 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:05.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:05 smithi134 bash[24068]: audit 2024-07-15T06:48:04.938720+0000 mon.a (mon.0) 699 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:05.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:05 smithi134 bash[24068]: audit 2024-07-15T06:48:04.942778+0000 mon.a (mon.0) 700 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:05.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:05 smithi134 bash[24068]: audit 2024-07-15T06:48:04.972977+0000 mon.a (mon.0) 701 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:48:05.619 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:05 smithi089 bash[31372]: cluster 2024-07-15T06:48:04.108829+0000 mgr.y (mgr.24272) 31 : cluster [DBG] pgmap v13: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:48:05.619 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:05 smithi089 bash[31372]: audit 2024-07-15T06:48:04.929804+0000 mon.a (mon.0) 697 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:05.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:05 smithi089 bash[31372]: audit 2024-07-15T06:48:04.934513+0000 mon.a (mon.0) 698 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:05.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:05 smithi089 bash[31372]: audit 2024-07-15T06:48:04.938720+0000 mon.a (mon.0) 699 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:05.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:05 smithi089 bash[31372]: audit 2024-07-15T06:48:04.942778+0000 mon.a (mon.0) 700 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:05.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:05 smithi089 bash[31372]: audit 2024-07-15T06:48:04.972977+0000 mon.a (mon.0) 701 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:48:05.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:05 smithi089 bash[21793]: cluster 2024-07-15T06:48:04.108829+0000 mgr.y (mgr.24272) 31 : cluster [DBG] pgmap v13: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:48:05.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:05 smithi089 bash[21793]: audit 2024-07-15T06:48:04.929804+0000 mon.a (mon.0) 697 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:05.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:05 smithi089 bash[21793]: audit 2024-07-15T06:48:04.934513+0000 mon.a (mon.0) 698 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:05.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:05 smithi089 bash[21793]: audit 2024-07-15T06:48:04.938720+0000 mon.a (mon.0) 699 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:05.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:05 smithi089 bash[21793]: audit 2024-07-15T06:48:04.942778+0000 mon.a (mon.0) 700 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:05.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:05 smithi089 bash[21793]: audit 2024-07-15T06:48:04.972977+0000 mon.a (mon.0) 701 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:48:06.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:06 smithi134 bash[24068]: audit 2024-07-15T06:48:05.219936+0000 mon.a (mon.0) 702 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:06.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:06 smithi134 bash[24068]: audit 2024-07-15T06:48:05.226079+0000 mon.a (mon.0) 703 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:06.573 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:06 smithi089 bash[21793]: audit 2024-07-15T06:48:05.219936+0000 mon.a (mon.0) 702 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:06.573 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:06 smithi089 bash[21793]: audit 2024-07-15T06:48:05.226079+0000 mon.a (mon.0) 703 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:06.573 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:06 smithi089 bash[31372]: audit 2024-07-15T06:48:05.219936+0000 mon.a (mon.0) 702 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:06.573 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:06 smithi089 bash[31372]: audit 2024-07-15T06:48:05.226079+0000 mon.a (mon.0) 703 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:06.723 INFO:teuthology.orchestra.run.smithi089.stdout:[client.0] 2024-07-15T06:48:06.724 INFO:teuthology.orchestra.run.smithi089.stdout: key = AQAmxpRm6S8zKxAAVM0CUf5B8q5IRysJkARFfQ== 2024-07-15T06:48:06.869 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 06:48:06 smithi089 bash[56397]: ts=2024-07-15T06:48:06.645Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-07-15T06:48:06.870 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 06:48:06 smithi089 bash[56397]: ts=2024-07-15T06:48:06.645Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-07-15T06:48:06.870 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 06:48:06 smithi089 bash[56397]: ts=2024-07-15T06:48:06.646Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.89 port=9094 2024-07-15T06:48:06.870 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 06:48:06 smithi089 bash[56397]: ts=2024-07-15T06:48:06.647Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-07-15T06:48:06.870 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 06:48:06 smithi089 bash[56397]: ts=2024-07-15T06:48:06.676Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-07-15T06:48:06.870 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 06:48:06 smithi089 bash[56397]: ts=2024-07-15T06:48:06.677Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-07-15T06:48:06.870 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 06:48:06 smithi089 bash[56397]: ts=2024-07-15T06:48:06.680Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-07-15T06:48:06.870 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 06:48:06 smithi089 bash[56397]: ts=2024-07-15T06:48:06.680Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-07-15T06:48:07.223 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:48:06 smithi134 bash[25019]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:48:06] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-07-15T06:48:07.377 DEBUG:teuthology.orchestra.run.smithi089:> set -ex 2024-07-15T06:48:07.378 DEBUG:teuthology.orchestra.run.smithi089:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-07-15T06:48:07.378 DEBUG:teuthology.orchestra.run.smithi089:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-07-15T06:48:07.402 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-07-15T06:48:07.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:07 smithi134 bash[24068]: cluster 2024-07-15T06:48:06.109454+0000 mgr.y (mgr.24272) 32 : cluster [DBG] pgmap v14: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:48:07.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:07 smithi134 bash[24068]: audit 2024-07-15T06:48:06.724127+0000 mon.c (mon.2) 16 : audit [INF] from='client.? 172.21.15.89:0/2433811138' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-07-15T06:48:07.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:07 smithi134 bash[24068]: audit 2024-07-15T06:48:06.724464+0000 mon.a (mon.0) 704 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-07-15T06:48:07.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:07 smithi134 bash[24068]: audit 2024-07-15T06:48:06.729079+0000 mon.a (mon.0) 705 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-07-15T06:48:07.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:07 smithi089 bash[21793]: cluster 2024-07-15T06:48:06.109454+0000 mgr.y (mgr.24272) 32 : cluster [DBG] pgmap v14: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:48:07.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:07 smithi089 bash[21793]: audit 2024-07-15T06:48:06.724127+0000 mon.c (mon.2) 16 : audit [INF] from='client.? 172.21.15.89:0/2433811138' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-07-15T06:48:07.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:07 smithi089 bash[21793]: audit 2024-07-15T06:48:06.724464+0000 mon.a (mon.0) 704 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-07-15T06:48:07.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:07 smithi089 bash[21793]: audit 2024-07-15T06:48:06.729079+0000 mon.a (mon.0) 705 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-07-15T06:48:07.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:07 smithi089 bash[31372]: cluster 2024-07-15T06:48:06.109454+0000 mgr.y (mgr.24272) 32 : cluster [DBG] pgmap v14: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:48:07.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:07 smithi089 bash[31372]: audit 2024-07-15T06:48:06.724127+0000 mon.c (mon.2) 16 : audit [INF] from='client.? 172.21.15.89:0/2433811138' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-07-15T06:48:07.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:07 smithi089 bash[31372]: audit 2024-07-15T06:48:06.724464+0000 mon.a (mon.0) 704 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-07-15T06:48:07.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:07 smithi089 bash[31372]: audit 2024-07-15T06:48:06.729079+0000 mon.a (mon.0) 705 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-07-15T06:48:09.119 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 06:48:08 smithi089 bash[56397]: ts=2024-07-15T06:48:08.647Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.00020416s 2024-07-15T06:48:09.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:09 smithi134 bash[24068]: cluster 2024-07-15T06:48:08.110021+0000 mgr.y (mgr.24272) 33 : cluster [DBG] pgmap v15: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:48:09.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:09 smithi134 bash[24068]: audit 2024-07-15T06:48:09.192381+0000 mon.a (mon.0) 706 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:09.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:09 smithi089 bash[21793]: cluster 2024-07-15T06:48:08.110021+0000 mgr.y (mgr.24272) 33 : cluster [DBG] pgmap v15: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:48:09.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:09 smithi089 bash[21793]: audit 2024-07-15T06:48:09.192381+0000 mon.a (mon.0) 706 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:09.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:09 smithi089 bash[31372]: cluster 2024-07-15T06:48:08.110021+0000 mgr.y (mgr.24272) 33 : cluster [DBG] pgmap v15: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:48:09.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:09 smithi089 bash[31372]: audit 2024-07-15T06:48:09.192381+0000 mon.a (mon.0) 706 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:11.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:11 smithi134 bash[24068]: cluster 2024-07-15T06:48:10.110616+0000 mgr.y (mgr.24272) 34 : cluster [DBG] pgmap v16: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:48:11.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:11 smithi134 bash[24068]: audit 2024-07-15T06:48:10.294382+0000 mon.a (mon.0) 707 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:11.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:11 smithi134 bash[24068]: audit 2024-07-15T06:48:10.303704+0000 mon.a (mon.0) 708 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:11.604 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:11 smithi089 bash[21793]: cluster 2024-07-15T06:48:10.110616+0000 mgr.y (mgr.24272) 34 : cluster [DBG] pgmap v16: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:48:11.604 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:11 smithi089 bash[21793]: audit 2024-07-15T06:48:10.294382+0000 mon.a (mon.0) 707 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:11.604 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:11 smithi089 bash[21793]: audit 2024-07-15T06:48:10.303704+0000 mon.a (mon.0) 708 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:11.604 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:11 smithi089 bash[31372]: cluster 2024-07-15T06:48:10.110616+0000 mgr.y (mgr.24272) 34 : cluster [DBG] pgmap v16: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:48:11.604 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:11 smithi089 bash[31372]: audit 2024-07-15T06:48:10.294382+0000 mon.a (mon.0) 707 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:11.604 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:11 smithi089 bash[31372]: audit 2024-07-15T06:48:10.303704+0000 mon.a (mon.0) 708 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:11.779 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.b/config 2024-07-15T06:48:11.869 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:48:11 smithi089 bash[22045]: [15/Jul/2024:06:48:11] ENGINE Bus STOPPING 2024-07-15T06:48:12.370 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:48:11 smithi089 bash[22045]: [15/Jul/2024:06:48:11] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-07-15T06:48:12.370 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:48:11 smithi089 bash[22045]: [15/Jul/2024:06:48:11] ENGINE Bus STOPPED 2024-07-15T06:48:12.370 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:48:11 smithi089 bash[22045]: [15/Jul/2024:06:48:11] ENGINE Bus STARTING 2024-07-15T06:48:12.370 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:48:11 smithi089 bash[22045]: [15/Jul/2024:06:48:11] ENGINE Serving on http://:::9283 2024-07-15T06:48:12.370 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:48:11 smithi089 bash[22045]: [15/Jul/2024:06:48:11] ENGINE Bus STARTED 2024-07-15T06:48:12.826 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:12 smithi134 bash[24068]: audit 2024-07-15T06:48:11.508919+0000 mon.a (mon.0) 709 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:12.827 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:12 smithi134 bash[24068]: audit 2024-07-15T06:48:11.513321+0000 mon.a (mon.0) 710 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:12.827 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:12 smithi134 bash[24068]: audit 2024-07-15T06:48:11.514118+0000 mon.a (mon.0) 711 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:48:12.827 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:12 smithi134 bash[24068]: audit 2024-07-15T06:48:11.514801+0000 mon.a (mon.0) 712 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:48:12.827 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:12 smithi134 bash[24068]: audit 2024-07-15T06:48:11.519583+0000 mon.a (mon.0) 713 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:12.827 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:12 smithi134 bash[24068]: audit 2024-07-15T06:48:11.583261+0000 mon.a (mon.0) 714 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:12.827 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:12 smithi134 bash[24068]: audit 2024-07-15T06:48:11.588791+0000 mon.a (mon.0) 715 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:12.827 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:12 smithi134 bash[24068]: audit 2024-07-15T06:48:11.592112+0000 mon.a (mon.0) 716 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-07-15T06:48:12.827 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:12 smithi134 bash[24068]: audit 2024-07-15T06:48:11.593020+0000 mgr.y (mgr.24272) 35 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-07-15T06:48:12.827 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:12 smithi134 bash[24068]: audit 2024-07-15T06:48:11.599829+0000 mon.a (mon.0) 717 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:12.827 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:12 smithi134 bash[24068]: cephadm 2024-07-15T06:48:11.618956+0000 mgr.y (mgr.24272) 36 : cephadm [INF] Deploying daemon grafana.a on smithi134 2024-07-15T06:48:12.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:12 smithi089 bash[21793]: audit 2024-07-15T06:48:11.508919+0000 mon.a (mon.0) 709 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:12.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:12 smithi089 bash[21793]: audit 2024-07-15T06:48:11.513321+0000 mon.a (mon.0) 710 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:12.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:12 smithi089 bash[21793]: audit 2024-07-15T06:48:11.514118+0000 mon.a (mon.0) 711 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:48:12.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:12 smithi089 bash[21793]: audit 2024-07-15T06:48:11.514801+0000 mon.a (mon.0) 712 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:48:12.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:12 smithi089 bash[21793]: audit 2024-07-15T06:48:11.519583+0000 mon.a (mon.0) 713 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:12.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:12 smithi089 bash[21793]: audit 2024-07-15T06:48:11.583261+0000 mon.a (mon.0) 714 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:12.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:12 smithi089 bash[21793]: audit 2024-07-15T06:48:11.588791+0000 mon.a (mon.0) 715 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:12.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:12 smithi089 bash[21793]: audit 2024-07-15T06:48:11.592112+0000 mon.a (mon.0) 716 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-07-15T06:48:12.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:12 smithi089 bash[21793]: audit 2024-07-15T06:48:11.593020+0000 mgr.y (mgr.24272) 35 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-07-15T06:48:12.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:12 smithi089 bash[21793]: audit 2024-07-15T06:48:11.599829+0000 mon.a (mon.0) 717 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:12.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:12 smithi089 bash[21793]: cephadm 2024-07-15T06:48:11.618956+0000 mgr.y (mgr.24272) 36 : cephadm [INF] Deploying daemon grafana.a on smithi134 2024-07-15T06:48:12.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:12 smithi089 bash[31372]: audit 2024-07-15T06:48:11.508919+0000 mon.a (mon.0) 709 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:12.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:12 smithi089 bash[31372]: audit 2024-07-15T06:48:11.513321+0000 mon.a (mon.0) 710 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:12.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:12 smithi089 bash[31372]: audit 2024-07-15T06:48:11.514118+0000 mon.a (mon.0) 711 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:48:12.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:12 smithi089 bash[31372]: audit 2024-07-15T06:48:11.514801+0000 mon.a (mon.0) 712 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:48:12.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:12 smithi089 bash[31372]: audit 2024-07-15T06:48:11.519583+0000 mon.a (mon.0) 713 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:12.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:12 smithi089 bash[31372]: audit 2024-07-15T06:48:11.583261+0000 mon.a (mon.0) 714 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:12.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:12 smithi089 bash[31372]: audit 2024-07-15T06:48:11.588791+0000 mon.a (mon.0) 715 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:12.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:12 smithi089 bash[31372]: audit 2024-07-15T06:48:11.592112+0000 mon.a (mon.0) 716 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-07-15T06:48:12.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:12 smithi089 bash[31372]: audit 2024-07-15T06:48:11.593020+0000 mgr.y (mgr.24272) 35 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-07-15T06:48:12.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:12 smithi089 bash[31372]: audit 2024-07-15T06:48:11.599829+0000 mon.a (mon.0) 717 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:12.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:12 smithi089 bash[31372]: cephadm 2024-07-15T06:48:11.618956+0000 mgr.y (mgr.24272) 36 : cephadm [INF] Deploying daemon grafana.a on smithi134 2024-07-15T06:48:13.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:13 smithi134 bash[24068]: cluster 2024-07-15T06:48:12.111216+0000 mgr.y (mgr.24272) 37 : cluster [DBG] pgmap v17: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:48:13.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:13 smithi089 bash[21793]: cluster 2024-07-15T06:48:12.111216+0000 mgr.y (mgr.24272) 37 : cluster [DBG] pgmap v17: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:48:13.870 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:13 smithi089 bash[31372]: cluster 2024-07-15T06:48:12.111216+0000 mgr.y (mgr.24272) 37 : cluster [DBG] pgmap v17: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:48:14.869 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:14 smithi089 bash[31372]: audit 2024-07-15T06:48:14.188177+0000 mon.a (mon.0) 718 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:48:14.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:14 smithi089 bash[21793]: audit 2024-07-15T06:48:14.188177+0000 mon.a (mon.0) 718 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:48:15.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:14 smithi134 bash[24068]: audit 2024-07-15T06:48:14.188177+0000 mon.a (mon.0) 718 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:48:15.869 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:15 smithi089 bash[31372]: cluster 2024-07-15T06:48:14.111789+0000 mgr.y (mgr.24272) 38 : cluster [DBG] pgmap v18: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:48:15.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:15 smithi089 bash[21793]: cluster 2024-07-15T06:48:14.111789+0000 mgr.y (mgr.24272) 38 : cluster [DBG] pgmap v18: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:48:16.008 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:15 smithi134 bash[24068]: cluster 2024-07-15T06:48:14.111789+0000 mgr.y (mgr.24272) 38 : cluster [DBG] pgmap v18: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:48:16.427 INFO:teuthology.orchestra.run.smithi134.stdout:[client.1] 2024-07-15T06:48:16.427 INFO:teuthology.orchestra.run.smithi134.stdout: key = AQAwxpRml1RZGRAAxIpduipLglMBklKPt12NOg== 2024-07-15T06:48:16.731 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:16 smithi134 bash[24068]: audit 2024-07-15T06:48:16.423764+0000 mon.b (mon.1) 29 : audit [INF] from='client.? 172.21.15.134:0/893462845' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-07-15T06:48:16.731 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:16 smithi134 bash[24068]: audit 2024-07-15T06:48:16.424965+0000 mon.a (mon.0) 719 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-07-15T06:48:16.731 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:16 smithi134 bash[24068]: audit 2024-07-15T06:48:16.432516+0000 mon.a (mon.0) 720 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-07-15T06:48:16.869 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 06:48:16 smithi089 bash[56397]: ts=2024-07-15T06:48:16.648Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.001835849s 2024-07-15T06:48:16.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:16 smithi089 bash[21793]: audit 2024-07-15T06:48:16.423764+0000 mon.b (mon.1) 29 : audit [INF] from='client.? 172.21.15.134:0/893462845' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-07-15T06:48:16.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:16 smithi089 bash[21793]: audit 2024-07-15T06:48:16.424965+0000 mon.a (mon.0) 719 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-07-15T06:48:16.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:16 smithi089 bash[21793]: audit 2024-07-15T06:48:16.432516+0000 mon.a (mon.0) 720 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-07-15T06:48:16.870 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:16 smithi089 bash[31372]: audit 2024-07-15T06:48:16.423764+0000 mon.b (mon.1) 29 : audit [INF] from='client.? 172.21.15.134:0/893462845' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-07-15T06:48:16.870 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:16 smithi089 bash[31372]: audit 2024-07-15T06:48:16.424965+0000 mon.a (mon.0) 719 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-07-15T06:48:16.870 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:16 smithi089 bash[31372]: audit 2024-07-15T06:48:16.432516+0000 mon.a (mon.0) 720 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-07-15T06:48:17.055 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:48:16 smithi134 bash[25019]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:48:16] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-07-15T06:48:18.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:17 smithi134 bash[24068]: cluster 2024-07-15T06:48:16.112386+0000 mgr.y (mgr.24272) 39 : cluster [DBG] pgmap v19: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:48:18.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:17 smithi089 bash[21793]: cluster 2024-07-15T06:48:16.112386+0000 mgr.y (mgr.24272) 39 : cluster [DBG] pgmap v19: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:48:18.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:17 smithi089 bash[31372]: cluster 2024-07-15T06:48:16.112386+0000 mgr.y (mgr.24272) 39 : cluster [DBG] pgmap v19: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:48:18.401 DEBUG:teuthology.orchestra.run.smithi134:> set -ex 2024-07-15T06:48:18.401 DEBUG:teuthology.orchestra.run.smithi134:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2024-07-15T06:48:18.401 DEBUG:teuthology.orchestra.run.smithi134:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2024-07-15T06:48:18.421 INFO:tasks.cephadm:Waiting for OSDs to come up 2024-07-15T06:48:18.422 DEBUG:teuthology.orchestra.run.smithi089:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster ceph osd dump --format=json 2024-07-15T06:48:18.719 INFO:teuthology.misc.health.smithi089.stdout: 2024-07-15T06:48:18.719 INFO:teuthology.misc.health.smithi089.stdout:{"epoch":59,"fsid":"75ba02bc-4274-11ef-bcad-c7b262605968","created":"2024-07-15T06:37:09.634877+0000","modified":"2024-07-15T06:47:44.086504+0000","last_up_change":"2024-07-15T06:46:45.306680+0000","last_in_change":"2024-07-15T06:46:19.019927+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-07-15T06:42:21.444347+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"79357114-251f-489d-bef5-f6ec37002212","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6802","nonce":843843653},{"type":"v1","addr":"172.21.15.89:6803","nonce":843843653}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6804","nonce":843843653},{"type":"v1","addr":"172.21.15.89:6805","nonce":843843653}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6808","nonce":843843653},{"type":"v1","addr":"172.21.15.89:6809","nonce":843843653}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6806","nonce":843843653},{"type":"v1","addr":"172.21.15.89:6807","nonce":843843653}]},"public_addr":"172.21.15.89:6803/843843653","cluster_addr":"172.21.15.89:6805/843843653","heartbeat_back_addr":"172.21.15.89:6809/843843653","heartbeat_front_addr":"172.21.15.89:6807/843843653","state":["exists","up"]},{"osd":1,"uuid":"d304859e-317a-414e-a16c-f097fc4b2702","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6810","nonce":1536075263},{"type":"v1","addr":"172.21.15.89:6811","nonce":1536075263}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6812","nonce":1536075263},{"type":"v1","addr":"172.21.15.89:6813","nonce":1536075263}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6816","nonce":1536075263},{"type":"v1","addr":"172.21.15.89:6817","nonce":1536075263}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6814","nonce":1536075263},{"type":"v1","addr":"172.21.15.89:6815","nonce":1536075263}]},"public_addr":"172.21.15.89:6811/1536075263","cluster_addr":"172.21.15.89:6813/1536075263","heartbeat_back_addr":"172.21.15.89:6817/1536075263","heartbeat_front_addr":"172.21.15.89:6815/1536075263","state":["exists","up"]},{"osd":2,"uuid":"60cde686-83bb-4ca9-a0f1-6668710e16f1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6818","nonce":3236432318},{"type":"v1","addr":"172.21.15.89:6819","nonce":3236432318}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6820","nonce":3236432318},{"type":"v1","addr":"172.21.15.89:6821","nonce":3236432318}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6824","nonce":3236432318},{"type":"v1","addr":"172.21.15.89:6825","nonce":3236432318}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6822","nonce":3236432318},{"type":"v1","addr":"172.21.15.89:6823","nonce":3236432318}]},"public_addr":"172.21.15.89:6819/3236432318","cluster_addr":"172.21.15.89:6821/3236432318","heartbeat_back_addr":"172.21.15.89:6825/3236432318","heartbeat_front_addr":"172.21.15.89:6823/3236432318","state":["exists","up"]},{"osd":3,"uuid":"8af52bbd-4bc9-4b13-b525-26c92e69c29f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6826","nonce":4083663609},{"type":"v1","addr":"172.21.15.89:6827","nonce":4083663609}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6828","nonce":4083663609},{"type":"v1","addr":"172.21.15.89:6829","nonce":4083663609}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6832","nonce":4083663609},{"type":"v1","addr":"172.21.15.89:6833","nonce":4083663609}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6830","nonce":4083663609},{"type":"v1","addr":"172.21.15.89:6831","nonce":4083663609}]},"public_addr":"172.21.15.89:6827/4083663609","cluster_addr":"172.21.15.89:6829/4083663609","heartbeat_back_addr":"172.21.15.89:6833/4083663609","heartbeat_front_addr":"172.21.15.89:6831/4083663609","state":["exists","up"]},{"osd":4,"uuid":"b97874a6-e72a-4068-9c71-a50b8790be2c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6800","nonce":3588917663},{"type":"v1","addr":"172.21.15.134:6801","nonce":3588917663}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6802","nonce":3588917663},{"type":"v1","addr":"172.21.15.134:6803","nonce":3588917663}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6806","nonce":3588917663},{"type":"v1","addr":"172.21.15.134:6807","nonce":3588917663}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6804","nonce":3588917663},{"type":"v1","addr":"172.21.15.134:6805","nonce":3588917663}]},"public_addr":"172.21.15.134:6801/3588917663","cluster_addr":"172.21.15.134:6803/3588917663","heartbeat_back_addr":"172.21.15.134:6807/3588917663","heartbeat_front_addr":"172.21.15.134:6805/3588917663","state":["exists","up"]},{"osd":5,"uuid":"a834c8ee-e034-423d-ab67-1ab33bc15a97","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":41,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6808","nonce":823577783},{"type":"v1","addr":"172.21.15.134:6809","nonce":823577783}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6810","nonce":823577783},{"type":"v1","addr":"172.21.15.134:6811","nonce":823577783}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6814","nonce":823577783},{"type":"v1","addr":"172.21.15.134:6815","nonce":823577783}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6812","nonce":823577783},{"type":"v1","addr":"172.21.15.134:6813","nonce":823577783}]},"public_addr":"172.21.15.134:6809/823577783","cluster_addr":"172.21.15.134:6811/823577783","heartbeat_back_addr":"172.21.15.134:6815/823577783","heartbeat_front_addr":"172.21.15.134:6813/823577783","state":["exists","up"]},{"osd":6,"uuid":"446cb8fa-8ddb-4f7a-a8a4-c7281e950f74","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6816","nonce":4010812142},{"type":"v1","addr":"172.21.15.134:6817","nonce":4010812142}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6818","nonce":4010812142},{"type":"v1","addr":"172.21.15.134:6819","nonce":4010812142}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6822","nonce":4010812142},{"type":"v1","addr":"172.21.15.134:6823","nonce":4010812142}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6820","nonce":4010812142},{"type":"v1","addr":"172.21.15.134:6821","nonce":4010812142}]},"public_addr":"172.21.15.134:6817/4010812142","cluster_addr":"172.21.15.134:6819/4010812142","heartbeat_back_addr":"172.21.15.134:6823/4010812142","heartbeat_front_addr":"172.21.15.134:6821/4010812142","state":["exists","up"]},{"osd":7,"uuid":"87f3a30f-2935-4b92-af37-bd866696838f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6824","nonce":3883480074},{"type":"v1","addr":"172.21.15.134:6825","nonce":3883480074}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6826","nonce":3883480074},{"type":"v1","addr":"172.21.15.134:6827","nonce":3883480074}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6830","nonce":3883480074},{"type":"v1","addr":"172.21.15.134:6831","nonce":3883480074}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6828","nonce":3883480074},{"type":"v1","addr":"172.21.15.134:6829","nonce":3883480074}]},"public_addr":"172.21.15.134:6825/3883480074","cluster_addr":"172.21.15.134:6827/3883480074","heartbeat_back_addr":"172.21.15.134:6831/3883480074","heartbeat_front_addr":"172.21.15.134:6829/3883480074","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-15T06:40:36.567653+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-15T06:41:26.849575+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-15T06:42:18.731018+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-15T06:43:11.604111+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-15T06:44:03.847505+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-15T06:44:56.895305+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-15T06:45:50.039730+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-15T06:46:43.462786+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.89:6801/265242087":"2024-07-16T06:47:44.086461+0000","172.21.15.89:0/1100221668":"2024-07-16T06:47:44.086461+0000","172.21.15.89:0/1877203535":"2024-07-16T06:37:35.981306+0000","172.21.15.89:0/1750227302":"2024-07-16T06:38:06.841135+0000","172.21.15.89:6800/2189595088":"2024-07-16T06:37:35.981306+0000","172.21.15.89:6801/2189595088":"2024-07-16T06:37:35.981306+0000","172.21.15.89:0/3465936199":"2024-07-16T06:47:44.086461+0000","172.21.15.89:0/1228522973":"2024-07-16T06:37:35.981306+0000","172.21.15.89:0/889381338":"2024-07-16T06:38:06.841135+0000","172.21.15.89:0/1672418864":"2024-07-16T06:38:06.841135+0000","172.21.15.89:0/2127823469":"2024-07-16T06:37:35.981306+0000","172.21.15.89:6801/2919301358":"2024-07-16T06:38:06.841135+0000","172.21.15.89:6800/265242087":"2024-07-16T06:47:44.086461+0000","172.21.15.89:6800/2919301358":"2024-07-16T06:38:06.841135+0000","172.21.15.89:0/3680199495":"2024-07-16T06:47:44.086461+0000","172.21.15.89:0/1054475873":"2024-07-16T06:47:44.086461+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-07-15T06:48:18.737 DEBUG:teuthology.misc:8 of 8 OSDs are up 2024-07-15T06:48:18.737 INFO:tasks.cephadm:Creating RBD pool 2024-07-15T06:48:18.737 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- sudo ceph --cluster ceph osd pool create rbd 8 2024-07-15T06:48:20.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:19 smithi134 bash[24068]: cluster 2024-07-15T06:48:18.112943+0000 mgr.y (mgr.24272) 40 : cluster [DBG] pgmap v20: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:48:20.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:19 smithi134 bash[24068]: audit 2024-07-15T06:48:18.724317+0000 mon.a (mon.0) 721 : audit [DBG] from='client.? 172.21.15.89:0/4139909756' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-07-15T06:48:20.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:19 smithi089 bash[21793]: cluster 2024-07-15T06:48:18.112943+0000 mgr.y (mgr.24272) 40 : cluster [DBG] pgmap v20: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:48:20.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:19 smithi089 bash[21793]: audit 2024-07-15T06:48:18.724317+0000 mon.a (mon.0) 721 : audit [DBG] from='client.? 172.21.15.89:0/4139909756' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-07-15T06:48:20.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:19 smithi089 bash[31372]: cluster 2024-07-15T06:48:18.112943+0000 mgr.y (mgr.24272) 40 : cluster [DBG] pgmap v20: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:48:20.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:19 smithi089 bash[31372]: audit 2024-07-15T06:48:18.724317+0000 mon.a (mon.0) 721 : audit [DBG] from='client.? 172.21.15.89:0/4139909756' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-07-15T06:48:22.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:21 smithi134 bash[24068]: cluster 2024-07-15T06:48:20.113570+0000 mgr.y (mgr.24272) 41 : cluster [DBG] pgmap v21: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:48:22.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:21 smithi089 bash[21793]: cluster 2024-07-15T06:48:20.113570+0000 mgr.y (mgr.24272) 41 : cluster [DBG] pgmap v21: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:48:22.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:21 smithi089 bash[31372]: cluster 2024-07-15T06:48:20.113570+0000 mgr.y (mgr.24272) 41 : cluster [DBG] pgmap v21: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:48:23.512 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T06:48:23.869 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:23 smithi089 bash[21793]: cluster 2024-07-15T06:48:22.114186+0000 mgr.y (mgr.24272) 42 : cluster [DBG] pgmap v22: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:48:23.870 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:23 smithi089 bash[31372]: cluster 2024-07-15T06:48:22.114186+0000 mgr.y (mgr.24272) 42 : cluster [DBG] pgmap v22: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:48:24.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:23 smithi134 bash[24068]: cluster 2024-07-15T06:48:22.114186+0000 mgr.y (mgr.24272) 42 : cluster [DBG] pgmap v22: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:48:25.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:25 smithi089 bash[21793]: cluster 2024-07-15T06:48:24.114719+0000 mgr.y (mgr.24272) 43 : cluster [DBG] pgmap v23: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:48:25.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:25 smithi089 bash[31372]: cluster 2024-07-15T06:48:24.114719+0000 mgr.y (mgr.24272) 43 : cluster [DBG] pgmap v23: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:48:26.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:25 smithi134 bash[24068]: cluster 2024-07-15T06:48:24.114719+0000 mgr.y (mgr.24272) 43 : cluster [DBG] pgmap v23: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:48:27.305 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:48:26 smithi134 bash[25019]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:48:26] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-07-15T06:48:27.664 INFO:teuthology.orchestra.run.smithi089.stderr:pool 'rbd' created 2024-07-15T06:48:28.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:27 smithi134 bash[24068]: cluster 2024-07-15T06:48:26.115356+0000 mgr.y (mgr.24272) 44 : cluster [DBG] pgmap v24: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:48:28.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:27 smithi134 bash[24068]: audit 2024-07-15T06:48:26.845138+0000 mon.c (mon.2) 17 : audit [INF] from='client.? 172.21.15.89:0/1315659072' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-07-15T06:48:28.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:27 smithi134 bash[24068]: audit 2024-07-15T06:48:26.845732+0000 mon.a (mon.0) 722 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-07-15T06:48:28.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:27 smithi089 bash[31372]: cluster 2024-07-15T06:48:26.115356+0000 mgr.y (mgr.24272) 44 : cluster [DBG] pgmap v24: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:48:28.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:27 smithi089 bash[31372]: audit 2024-07-15T06:48:26.845138+0000 mon.c (mon.2) 17 : audit [INF] from='client.? 172.21.15.89:0/1315659072' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-07-15T06:48:28.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:27 smithi089 bash[31372]: audit 2024-07-15T06:48:26.845732+0000 mon.a (mon.0) 722 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-07-15T06:48:28.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:27 smithi089 bash[21793]: cluster 2024-07-15T06:48:26.115356+0000 mgr.y (mgr.24272) 44 : cluster [DBG] pgmap v24: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:48:28.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:27 smithi089 bash[21793]: audit 2024-07-15T06:48:26.845138+0000 mon.c (mon.2) 17 : audit [INF] from='client.? 172.21.15.89:0/1315659072' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-07-15T06:48:28.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:27 smithi089 bash[21793]: audit 2024-07-15T06:48:26.845732+0000 mon.a (mon.0) 722 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-07-15T06:48:28.319 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- sudo ceph --cluster ceph osd pool application enable rbd rbd --yes-i-really-mean-it 2024-07-15T06:48:29.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:28 smithi134 bash[24068]: audit 2024-07-15T06:48:27.668325+0000 mon.a (mon.0) 723 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-07-15T06:48:29.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:28 smithi134 bash[24068]: cluster 2024-07-15T06:48:27.668448+0000 mon.a (mon.0) 724 : cluster [DBG] osdmap e60: 8 total, 8 up, 8 in 2024-07-15T06:48:29.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:28 smithi134 bash[24068]: cluster 2024-07-15T06:48:28.666235+0000 mon.a (mon.0) 725 : cluster [DBG] osdmap e61: 8 total, 8 up, 8 in 2024-07-15T06:48:29.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:28 smithi089 bash[21793]: audit 2024-07-15T06:48:27.668325+0000 mon.a (mon.0) 723 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-07-15T06:48:29.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:28 smithi089 bash[21793]: cluster 2024-07-15T06:48:27.668448+0000 mon.a (mon.0) 724 : cluster [DBG] osdmap e60: 8 total, 8 up, 8 in 2024-07-15T06:48:29.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:28 smithi089 bash[21793]: cluster 2024-07-15T06:48:28.666235+0000 mon.a (mon.0) 725 : cluster [DBG] osdmap e61: 8 total, 8 up, 8 in 2024-07-15T06:48:29.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:28 smithi089 bash[31372]: audit 2024-07-15T06:48:27.668325+0000 mon.a (mon.0) 723 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-07-15T06:48:29.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:28 smithi089 bash[31372]: cluster 2024-07-15T06:48:27.668448+0000 mon.a (mon.0) 724 : cluster [DBG] osdmap e60: 8 total, 8 up, 8 in 2024-07-15T06:48:29.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:28 smithi089 bash[31372]: cluster 2024-07-15T06:48:28.666235+0000 mon.a (mon.0) 725 : cluster [DBG] osdmap e61: 8 total, 8 up, 8 in 2024-07-15T06:48:30.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:29 smithi134 bash[24068]: cluster 2024-07-15T06:48:28.116039+0000 mgr.y (mgr.24272) 45 : cluster [DBG] pgmap v26: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:48:30.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:29 smithi134 bash[24068]: cluster 2024-07-15T06:48:28.672152+0000 mon.a (mon.0) 726 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-07-15T06:48:30.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:29 smithi134 bash[24068]: audit 2024-07-15T06:48:29.196816+0000 mon.a (mon.0) 727 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:30.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:29 smithi134 bash[24068]: audit 2024-07-15T06:48:29.198586+0000 mon.a (mon.0) 728 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:48:30.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:29 smithi134 bash[24068]: cluster 2024-07-15T06:48:29.667840+0000 mon.a (mon.0) 729 : cluster [DBG] osdmap e62: 8 total, 8 up, 8 in 2024-07-15T06:48:30.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:29 smithi089 bash[31372]: cluster 2024-07-15T06:48:28.116039+0000 mgr.y (mgr.24272) 45 : cluster [DBG] pgmap v26: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:48:30.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:29 smithi089 bash[31372]: cluster 2024-07-15T06:48:28.672152+0000 mon.a (mon.0) 726 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-07-15T06:48:30.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:29 smithi089 bash[31372]: audit 2024-07-15T06:48:29.196816+0000 mon.a (mon.0) 727 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:30.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:29 smithi089 bash[31372]: audit 2024-07-15T06:48:29.198586+0000 mon.a (mon.0) 728 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:48:30.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:29 smithi089 bash[31372]: cluster 2024-07-15T06:48:29.667840+0000 mon.a (mon.0) 729 : cluster [DBG] osdmap e62: 8 total, 8 up, 8 in 2024-07-15T06:48:30.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:29 smithi089 bash[21793]: cluster 2024-07-15T06:48:28.116039+0000 mgr.y (mgr.24272) 45 : cluster [DBG] pgmap v26: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:48:30.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:29 smithi089 bash[21793]: cluster 2024-07-15T06:48:28.672152+0000 mon.a (mon.0) 726 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-07-15T06:48:30.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:29 smithi089 bash[21793]: audit 2024-07-15T06:48:29.196816+0000 mon.a (mon.0) 727 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:30.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:29 smithi089 bash[21793]: audit 2024-07-15T06:48:29.198586+0000 mon.a (mon.0) 728 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:48:30.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:29 smithi089 bash[21793]: cluster 2024-07-15T06:48:29.667840+0000 mon.a (mon.0) 729 : cluster [DBG] osdmap e62: 8 total, 8 up, 8 in 2024-07-15T06:48:32.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:31 smithi134 bash[24068]: cluster 2024-07-15T06:48:30.116673+0000 mgr.y (mgr.24272) 46 : cluster [DBG] pgmap v29: 9 pgs: 7 unknown, 2 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:48:32.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:31 smithi089 bash[31372]: cluster 2024-07-15T06:48:30.116673+0000 mgr.y (mgr.24272) 46 : cluster [DBG] pgmap v29: 9 pgs: 7 unknown, 2 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:48:32.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:31 smithi089 bash[21793]: cluster 2024-07-15T06:48:30.116673+0000 mgr.y (mgr.24272) 46 : cluster [DBG] pgmap v29: 9 pgs: 7 unknown, 2 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:48:33.090 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T06:48:34.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:33 smithi134 bash[24068]: cluster 2024-07-15T06:48:32.117306+0000 mgr.y (mgr.24272) 47 : cluster [DBG] pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:48:34.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:33 smithi089 bash[21793]: cluster 2024-07-15T06:48:32.117306+0000 mgr.y (mgr.24272) 47 : cluster [DBG] pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:48:34.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:33 smithi089 bash[31372]: cluster 2024-07-15T06:48:32.117306+0000 mgr.y (mgr.24272) 47 : cluster [DBG] pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:48:36.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:35 smithi134 bash[24068]: cluster 2024-07-15T06:48:34.117811+0000 mgr.y (mgr.24272) 48 : cluster [DBG] pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:48:36.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:35 smithi089 bash[21793]: cluster 2024-07-15T06:48:34.117811+0000 mgr.y (mgr.24272) 48 : cluster [DBG] pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:48:36.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:35 smithi089 bash[31372]: cluster 2024-07-15T06:48:34.117811+0000 mgr.y (mgr.24272) 48 : cluster [DBG] pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:48:36.706 INFO:teuthology.orchestra.run.smithi089.stderr:enabled application 'rbd' on pool 'rbd' 2024-07-15T06:48:36.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:36 smithi134 bash[24068]: audit 2024-07-15T06:48:36.535801+0000 mon.c (mon.2) 18 : audit [INF] from='client.? 172.21.15.89:0/542757127' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2024-07-15T06:48:36.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:36 smithi134 bash[24068]: audit 2024-07-15T06:48:36.536328+0000 mon.a (mon.0) 730 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2024-07-15T06:48:36.983 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:36 smithi089 bash[31372]: audit 2024-07-15T06:48:36.535801+0000 mon.c (mon.2) 18 : audit [INF] from='client.? 172.21.15.89:0/542757127' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2024-07-15T06:48:36.983 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:36 smithi089 bash[31372]: audit 2024-07-15T06:48:36.536328+0000 mon.a (mon.0) 730 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2024-07-15T06:48:36.983 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:36 smithi089 bash[21793]: audit 2024-07-15T06:48:36.535801+0000 mon.c (mon.2) 18 : audit [INF] from='client.? 172.21.15.89:0/542757127' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2024-07-15T06:48:36.984 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:36 smithi089 bash[21793]: audit 2024-07-15T06:48:36.536328+0000 mon.a (mon.0) 730 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2024-07-15T06:48:37.305 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:48:36 smithi134 bash[25019]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:48:36] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-07-15T06:48:37.475 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-07-15T06:48:37.476 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-07-15T06:48:37.476 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph mgr dump --format=json 2024-07-15T06:48:37.869 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:37 smithi089 bash[21793]: cluster 2024-07-15T06:48:36.118512+0000 mgr.y (mgr.24272) 49 : cluster [DBG] pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:48:37.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:37 smithi089 bash[21793]: audit 2024-07-15T06:48:36.709950+0000 mon.a (mon.0) 731 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2024-07-15T06:48:37.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:37 smithi089 bash[21793]: cluster 2024-07-15T06:48:36.710085+0000 mon.a (mon.0) 732 : cluster [DBG] osdmap e63: 8 total, 8 up, 8 in 2024-07-15T06:48:37.870 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:37 smithi089 bash[31372]: cluster 2024-07-15T06:48:36.118512+0000 mgr.y (mgr.24272) 49 : cluster [DBG] pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:48:37.870 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:37 smithi089 bash[31372]: audit 2024-07-15T06:48:36.709950+0000 mon.a (mon.0) 731 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2024-07-15T06:48:37.870 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:37 smithi089 bash[31372]: cluster 2024-07-15T06:48:36.710085+0000 mon.a (mon.0) 732 : cluster [DBG] osdmap e63: 8 total, 8 up, 8 in 2024-07-15T06:48:38.002 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:37 smithi134 bash[24068]: cluster 2024-07-15T06:48:36.118512+0000 mgr.y (mgr.24272) 49 : cluster [DBG] pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:48:38.002 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:37 smithi134 bash[24068]: audit 2024-07-15T06:48:36.709950+0000 mon.a (mon.0) 731 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2024-07-15T06:48:38.003 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:37 smithi134 bash[24068]: cluster 2024-07-15T06:48:36.710085+0000 mon.a (mon.0) 732 : cluster [DBG] osdmap e63: 8 total, 8 up, 8 in 2024-07-15T06:48:38.710 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:38 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:48:38.710 INFO:journalctl@ceph.osd.4.smithi134.stdout:Jul 15 06:48:38 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:48:38.710 INFO:journalctl@ceph.osd.5.smithi134.stdout:Jul 15 06:48:38 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:48:38.711 INFO:journalctl@ceph.osd.6.smithi134.stdout:Jul 15 06:48:38 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:48:38.711 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:48:38 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:48:38.711 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:48:38 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:48:38.711 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:48:38 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:48:38.712 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 06:48:38 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:48:38.983 INFO:journalctl@ceph.osd.4.smithi134.stdout:Jul 15 06:48:38 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:48:38.983 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 06:48:38 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:48:38.984 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:48:38 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:48:38.984 INFO:journalctl@ceph.osd.5.smithi134.stdout:Jul 15 06:48:38 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:48:38.984 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:48:38 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:48:38.984 INFO:journalctl@ceph.osd.6.smithi134.stdout:Jul 15 06:48:38 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:48:38.985 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:38 smithi134 bash[24068]: cluster 2024-07-15T06:48:38.706672+0000 mon.a (mon.0) 733 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-07-15T06:48:38.985 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:38 smithi134 bash[24068]: cluster 2024-07-15T06:48:38.706735+0000 mon.a (mon.0) 734 : cluster [INF] Cluster is now healthy 2024-07-15T06:48:38.985 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:38 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:48:38.985 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:48:38 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:48:39.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:38 smithi089 bash[31372]: cluster 2024-07-15T06:48:38.706672+0000 mon.a (mon.0) 733 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-07-15T06:48:39.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:38 smithi089 bash[31372]: cluster 2024-07-15T06:48:38.706735+0000 mon.a (mon.0) 734 : cluster [INF] Cluster is now healthy 2024-07-15T06:48:39.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:38 smithi089 bash[21793]: cluster 2024-07-15T06:48:38.706672+0000 mon.a (mon.0) 733 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-07-15T06:48:39.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:38 smithi089 bash[21793]: cluster 2024-07-15T06:48:38.706735+0000 mon.a (mon.0) 734 : cluster [INF] Cluster is now healthy 2024-07-15T06:48:40.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:39 smithi134 bash[24068]: cluster 2024-07-15T06:48:38.119070+0000 mgr.y (mgr.24272) 50 : cluster [DBG] pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:48:40.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:39 smithi134 bash[24068]: audit 2024-07-15T06:48:39.031123+0000 mon.a (mon.0) 735 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:40.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:39 smithi134 bash[24068]: audit 2024-07-15T06:48:39.042713+0000 mon.a (mon.0) 736 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:40.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:39 smithi134 bash[24068]: audit 2024-07-15T06:48:39.052504+0000 mon.a (mon.0) 737 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:40.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:39 smithi134 bash[24068]: audit 2024-07-15T06:48:39.060835+0000 mon.a (mon.0) 738 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:40.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:39 smithi134 bash[24068]: audit 2024-07-15T06:48:39.118006+0000 mon.a (mon.0) 739 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:48:40.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:39 smithi134 bash[24068]: audit 2024-07-15T06:48:39.208926+0000 mon.a (mon.0) 740 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:40.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:39 smithi089 bash[21793]: cluster 2024-07-15T06:48:38.119070+0000 mgr.y (mgr.24272) 50 : cluster [DBG] pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:48:40.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:39 smithi089 bash[21793]: audit 2024-07-15T06:48:39.031123+0000 mon.a (mon.0) 735 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:40.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:39 smithi089 bash[21793]: audit 2024-07-15T06:48:39.042713+0000 mon.a (mon.0) 736 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:40.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:39 smithi089 bash[21793]: audit 2024-07-15T06:48:39.052504+0000 mon.a (mon.0) 737 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:40.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:39 smithi089 bash[21793]: audit 2024-07-15T06:48:39.060835+0000 mon.a (mon.0) 738 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:40.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:39 smithi089 bash[21793]: audit 2024-07-15T06:48:39.118006+0000 mon.a (mon.0) 739 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:48:40.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:39 smithi089 bash[21793]: audit 2024-07-15T06:48:39.208926+0000 mon.a (mon.0) 740 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:40.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:39 smithi089 bash[31372]: cluster 2024-07-15T06:48:38.119070+0000 mgr.y (mgr.24272) 50 : cluster [DBG] pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:48:40.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:39 smithi089 bash[31372]: audit 2024-07-15T06:48:39.031123+0000 mon.a (mon.0) 735 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:40.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:39 smithi089 bash[31372]: audit 2024-07-15T06:48:39.042713+0000 mon.a (mon.0) 736 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:40.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:39 smithi089 bash[31372]: audit 2024-07-15T06:48:39.052504+0000 mon.a (mon.0) 737 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:40.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:39 smithi089 bash[31372]: audit 2024-07-15T06:48:39.060835+0000 mon.a (mon.0) 738 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:40.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:39 smithi089 bash[31372]: audit 2024-07-15T06:48:39.118006+0000 mon.a (mon.0) 739 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:48:40.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:39 smithi089 bash[31372]: audit 2024-07-15T06:48:39.208926+0000 mon.a (mon.0) 740 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:40.656 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:40 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:40.618976702Z level=info msg="Executing migration" id="create user table v2" 2024-07-15T06:48:40.656 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:40 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:40.66096823Z level=info msg="Executing migration" id="create index UQE_user_login - v2" 2024-07-15T06:48:40.917 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:40 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:40.686081741Z level=info msg="Executing migration" id="create index UQE_user_email - v2" 2024-07-15T06:48:40.917 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:40 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:40.711560834Z level=info msg="Executing migration" id="copy data_source v1 to v2" 2024-07-15T06:48:40.917 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:40 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:40.736674072Z level=info msg="Executing migration" id="Drop old table user_v1" 2024-07-15T06:48:40.917 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:40 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:40.753619071Z level=info msg="Executing migration" id="Add column help_flags1 to user table" 2024-07-15T06:48:40.917 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:40 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:40.787180767Z level=info msg="Executing migration" id="Update user table charset" 2024-07-15T06:48:40.917 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:40 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:40.804180299Z level=info msg="Executing migration" id="Add last_seen_at column to user" 2024-07-15T06:48:40.917 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:40 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:40.829408248Z level=info msg="Executing migration" id="Add missing user data" 2024-07-15T06:48:40.917 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:40 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:40.896395825Z level=info msg="Executing migration" id="Add is_disabled column to user" 2024-07-15T06:48:40.918 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:40 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:40.921643963Z level=info msg="Executing migration" id="Add index user.login/user.email" 2024-07-15T06:48:41.186 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:40 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:40.946881709Z level=info msg="Executing migration" id="Add is_service_account column to user" 2024-07-15T06:48:41.186 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:40 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:40.972077994Z level=info msg="Executing migration" id="Update is_service_account column to nullable" 2024-07-15T06:48:41.186 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:40 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:40.99741504Z level=info msg="Executing migration" id="create temp user table v1-7" 2024-07-15T06:48:41.186 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:41 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:41.022636974Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v1-7" 2024-07-15T06:48:41.186 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:41 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:41.047954797Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v1-7" 2024-07-15T06:48:41.187 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:41 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:41.073133095Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v1-7" 2024-07-15T06:48:41.187 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:41 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:41.098335424Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v1-7" 2024-07-15T06:48:41.187 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:41 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:41.123599263Z level=info msg="Executing migration" id="Update temp_user table charset" 2024-07-15T06:48:41.187 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:41 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:41.148735901Z level=info msg="Executing migration" id="drop index IDX_temp_user_email - v1" 2024-07-15T06:48:41.187 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:41 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:41.19077119Z level=info msg="Executing migration" id="drop index IDX_temp_user_org_id - v1" 2024-07-15T06:48:41.447 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:41 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:41.232668925Z level=info msg="Executing migration" id="drop index IDX_temp_user_code - v1" 2024-07-15T06:48:41.447 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:41 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:41.274716297Z level=info msg="Executing migration" id="drop index IDX_temp_user_status - v1" 2024-07-15T06:48:41.447 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:41 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:41.299997475Z level=info msg="Executing migration" id="Rename table temp_user to temp_user_tmp_qwerty - v1" 2024-07-15T06:48:41.447 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:41 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:41.325263204Z level=info msg="Executing migration" id="create temp_user v2" 2024-07-15T06:48:41.447 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:41 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:41.367244041Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v2" 2024-07-15T06:48:41.447 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:41 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:41.400960029Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v2" 2024-07-15T06:48:41.447 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:41 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:41.426086303Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v2" 2024-07-15T06:48:41.447 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:41 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:41.451426527Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v2" 2024-07-15T06:48:41.707 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:41 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:41.476766264Z level=info msg="Executing migration" id="copy temp_user v1 to v2" 2024-07-15T06:48:41.708 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:41 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:41.501899136Z level=info msg="Executing migration" id="drop temp_user_tmp_qwerty" 2024-07-15T06:48:41.708 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:41 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:41.552513601Z level=info msg="Executing migration" id="Set created for temp users that will otherwise prematurely expire" 2024-07-15T06:48:41.708 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:41 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:41.594482965Z level=info msg="Executing migration" id="create star table" 2024-07-15T06:48:41.708 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:41 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:41.619763147Z level=info msg="Executing migration" id="add unique index star.user_id_dashboard_id" 2024-07-15T06:48:41.708 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:41 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:41.645061002Z level=info msg="Executing migration" id="create org table v1" 2024-07-15T06:48:41.708 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:41 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:41.670197186Z level=info msg="Executing migration" id="create index UQE_org_name - v1" 2024-07-15T06:48:41.708 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:41 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:41.712260331Z level=info msg="Executing migration" id="create org_user table v1" 2024-07-15T06:48:41.959 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:41 smithi134 bash[24068]: cluster 2024-07-15T06:48:40.119616+0000 mgr.y (mgr.24272) 51 : cluster [DBG] pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:48:41.959 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:41 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:41.754251874Z level=info msg="Executing migration" id="create index IDX_org_user_org_id - v1" 2024-07-15T06:48:41.959 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:41 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:41.79630576Z level=info msg="Executing migration" id="create index UQE_org_user_org_id_user_id - v1" 2024-07-15T06:48:41.959 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:41 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:41.821590555Z level=info msg="Executing migration" id="create index IDX_org_user_user_id - v1" 2024-07-15T06:48:41.960 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:41 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:41.846807608Z level=info msg="Executing migration" id="Update org table charset" 2024-07-15T06:48:41.960 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:41 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:41.871983928Z level=info msg="Executing migration" id="Update org_user table charset" 2024-07-15T06:48:41.960 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:41 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:41.913812858Z level=info msg="Executing migration" id="Migrate all Read Only Viewers to Viewers" 2024-07-15T06:48:41.960 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:41 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:41.939254179Z level=info msg="Executing migration" id="create dashboard table" 2024-07-15T06:48:41.960 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:41 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:41.964330988Z level=info msg="Executing migration" id="add index dashboard.account_id" 2024-07-15T06:48:42.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:41 smithi089 bash[31372]: cluster 2024-07-15T06:48:40.119616+0000 mgr.y (mgr.24272) 51 : cluster [DBG] pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:48:42.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:41 smithi089 bash[21793]: cluster 2024-07-15T06:48:40.119616+0000 mgr.y (mgr.24272) 51 : cluster [DBG] pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:48:42.242 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T06:48:42.245 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:41 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:41.989707694Z level=info msg="Executing migration" id="add unique index dashboard_account_id_slug" 2024-07-15T06:48:42.245 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:42 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:42.056801893Z level=info msg="Executing migration" id="create dashboard_tag table" 2024-07-15T06:48:42.245 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:42 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:42.082050939Z level=info msg="Executing migration" id="add unique index dashboard_tag.dasboard_id_term" 2024-07-15T06:48:42.245 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:42 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:42.107366518Z level=info msg="Executing migration" id="drop index UQE_dashboard_tag_dashboard_id_term - v1" 2024-07-15T06:48:42.245 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:42 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:42.132590412Z level=info msg="Executing migration" id="Rename table dashboard to dashboard_v1 - v1" 2024-07-15T06:48:42.246 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:42 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:42.174606433Z level=info msg="Executing migration" id="create dashboard v2" 2024-07-15T06:48:42.246 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:42 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:42.208182145Z level=info msg="Executing migration" id="create index IDX_dashboard_org_id - v2" 2024-07-15T06:48:42.246 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:42 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:42.250107911Z level=info msg="Executing migration" id="create index UQE_dashboard_org_id_slug - v2" 2024-07-15T06:48:42.498 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:42 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:42.292154032Z level=info msg="Executing migration" id="copy dashboard v1 to v2" 2024-07-15T06:48:42.498 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:42 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:42.334126884Z level=info msg="Executing migration" id="drop table dashboard_v1" 2024-07-15T06:48:42.498 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:42 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:42.359423353Z level=info msg="Executing migration" id="alter dashboard.data to mediumtext v1" 2024-07-15T06:48:42.498 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:42 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:42.401399848Z level=info msg="Executing migration" id="Add column updated_by in dashboard - v2" 2024-07-15T06:48:42.498 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:42 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:42.426622155Z level=info msg="Executing migration" id="Add column created_by in dashboard - v2" 2024-07-15T06:48:42.498 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:42 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:42.451851426Z level=info msg="Executing migration" id="Add column gnetId in dashboard" 2024-07-15T06:48:42.499 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:42 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:42.477443989Z level=info msg="Executing migration" id="Add index for gnetId in dashboard" 2024-07-15T06:48:42.499 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:42 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:42.502718525Z level=info msg="Executing migration" id="Add column plugin_id in dashboard" 2024-07-15T06:48:42.750 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:42 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:42.527921739Z level=info msg="Executing migration" id="Add index for plugin_id in dashboard" 2024-07-15T06:48:42.750 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:42 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:42.553171503Z level=info msg="Executing migration" id="Add index for dashboard_id in dashboard_tag" 2024-07-15T06:48:42.750 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:42 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:42.595047417Z level=info msg="Executing migration" id="Update dashboard table charset" 2024-07-15T06:48:42.750 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:42 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:42.620273974Z level=info msg="Executing migration" id="Update dashboard_tag table charset" 2024-07-15T06:48:42.750 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:42 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:42.662263162Z level=info msg="Executing migration" id="Add column folder_id in dashboard" 2024-07-15T06:48:42.750 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:42 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:42.704363001Z level=info msg="Executing migration" id="Add column isFolder in dashboard" 2024-07-15T06:48:42.751 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:42 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:42.729503921Z level=info msg="Executing migration" id="Add column has_acl in dashboard" 2024-07-15T06:48:42.751 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:42 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:42.754791673Z level=info msg="Executing migration" id="Add column uid in dashboard" 2024-07-15T06:48:43.011 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:42 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:42.796811148Z level=info msg="Executing migration" id="Update uid column values in dashboard" 2024-07-15T06:48:43.011 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:42 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:42.843110794Z level=info msg="Executing migration" id="Add unique index dashboard_org_id_uid" 2024-07-15T06:48:43.011 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:42 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:42.864264945Z level=info msg="Executing migration" id="Remove unique index org_id_slug" 2024-07-15T06:48:43.011 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:42 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:42.914398489Z level=info msg="Executing migration" id="Update dashboard title length" 2024-07-15T06:48:43.011 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:42 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:42.939689109Z level=info msg="Executing migration" id="Add unique index for dashboard_org_id_title_folder_id" 2024-07-15T06:48:43.011 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:42 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:42.964785677Z level=info msg="Executing migration" id="create dashboard_provisioning" 2024-07-15T06:48:43.011 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:42 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:42.990194169Z level=info msg="Executing migration" id="Rename table dashboard_provisioning to dashboard_provisioning_tmp_qwerty - v1" 2024-07-15T06:48:43.011 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:43 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:43.015411879Z level=info msg="Executing migration" id="create dashboard_provisioning v2" 2024-07-15T06:48:43.284 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:43 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:43.040735649Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id - v2" 2024-07-15T06:48:43.284 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:43 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:43.065946233Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id_name - v2" 2024-07-15T06:48:43.284 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:43 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:43.091210297Z level=info msg="Executing migration" id="copy dashboard_provisioning v1 to v2" 2024-07-15T06:48:43.285 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:43 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:43.116409869Z level=info msg="Executing migration" id="drop dashboard_provisioning_tmp_qwerty" 2024-07-15T06:48:43.285 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:43 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:43.141722892Z level=info msg="Executing migration" id="Add check_sum column" 2024-07-15T06:48:43.285 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:43 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:43.183567672Z level=info msg="Executing migration" id="Add index for dashboard_title" 2024-07-15T06:48:43.285 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:43 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:43.208874079Z level=info msg="Executing migration" id="delete tags for deleted dashboards" 2024-07-15T06:48:43.285 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:43 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:43.234013349Z level=info msg="Executing migration" id="delete stars for deleted dashboards" 2024-07-15T06:48:43.285 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:43 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:43.288923016Z level=info msg="Executing migration" id="Add index for dashboard_is_folder" 2024-07-15T06:48:43.555 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:43 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:43.31798081Z level=info msg="Executing migration" id="Add isPublic for dashboard" 2024-07-15T06:48:43.555 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:43 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:43.343321535Z level=info msg="Executing migration" id="create data_source table" 2024-07-15T06:48:43.555 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:43 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:43.385227257Z level=info msg="Executing migration" id="add index data_source.account_id" 2024-07-15T06:48:43.555 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:43 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:43.435745903Z level=info msg="Executing migration" id="add unique index data_source.account_id_name" 2024-07-15T06:48:43.555 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:43 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:43.461290133Z level=info msg="Executing migration" id="drop index IDX_data_source_account_id - v1" 2024-07-15T06:48:43.555 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:43 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:43.48645203Z level=info msg="Executing migration" id="drop index UQE_data_source_account_id_name - v1" 2024-07-15T06:48:43.556 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:43 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:43.511692664Z level=info msg="Executing migration" id="Rename table data_source to data_source_v1 - v1" 2024-07-15T06:48:43.556 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:43 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:43.536996341Z level=info msg="Executing migration" id="create data_source table v2" 2024-07-15T06:48:43.843 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:43 smithi134 bash[24068]: cluster 2024-07-15T06:48:42.120327+0000 mgr.y (mgr.24272) 52 : cluster [DBG] pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:48:43.843 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:43 smithi134 bash[24068]: audit 2024-07-15T06:48:43.750820+0000 mon.a (mon.0) 741 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:43.843 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:43 smithi134 bash[24068]: audit 2024-07-15T06:48:43.757692+0000 mon.a (mon.0) 742 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:43.843 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:43 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:43.578960811Z level=info msg="Executing migration" id="create index IDX_data_source_org_id - v2" 2024-07-15T06:48:43.843 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:43 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:43.604154937Z level=info msg="Executing migration" id="create index UQE_data_source_org_id_name - v2" 2024-07-15T06:48:43.843 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:43 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:43.646135074Z level=info msg="Executing migration" id="Drop old table data_source_v1 #2" 2024-07-15T06:48:43.844 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:43 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:43.688058652Z level=info msg="Executing migration" id="Add column with_credentials" 2024-07-15T06:48:43.844 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:43 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:43.71324669Z level=info msg="Executing migration" id="Add secure json data column" 2024-07-15T06:48:43.844 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:43 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:43.738592116Z level=info msg="Executing migration" id="Update data_source table charset" 2024-07-15T06:48:43.844 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:43 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:43.780564407Z level=info msg="Executing migration" id="Update initial version to 1" 2024-07-15T06:48:43.844 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:43 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:43.80585624Z level=info msg="Executing migration" id="Add read_only data column" 2024-07-15T06:48:44.112 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:43 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:43.847805158Z level=info msg="Executing migration" id="Migrate logging ds to loki ds" 2024-07-15T06:48:44.112 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:43 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:43.889777591Z level=info msg="Executing migration" id="Update json_data with nulls" 2024-07-15T06:48:44.112 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:43 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:43.915094908Z level=info msg="Executing migration" id="Add uid column" 2024-07-15T06:48:44.112 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:43 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:43.940242284Z level=info msg="Executing migration" id="Update uid value" 2024-07-15T06:48:44.112 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:43 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:43.957106351Z level=info msg="Executing migration" id="Add unique index datasource_org_id_uid" 2024-07-15T06:48:44.112 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:43 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:43.973898454Z level=info msg="Executing migration" id="add unique index datasource_org_id_is_default" 2024-07-15T06:48:44.112 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:43 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:43.999206662Z level=info msg="Executing migration" id="create api_key table" 2024-07-15T06:48:44.113 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:44 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:44.041191963Z level=info msg="Executing migration" id="add index api_key.account_id" 2024-07-15T06:48:44.113 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:44 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:44.066583613Z level=info msg="Executing migration" id="add index api_key.key" 2024-07-15T06:48:44.113 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:44 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:44.091882582Z level=info msg="Executing migration" id="add index api_key.account_id_name" 2024-07-15T06:48:44.113 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:44 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:44.117042536Z level=info msg="Executing migration" id="drop index IDX_api_key_account_id - v1" 2024-07-15T06:48:44.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:43 smithi089 bash[21793]: cluster 2024-07-15T06:48:42.120327+0000 mgr.y (mgr.24272) 52 : cluster [DBG] pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:48:44.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:43 smithi089 bash[21793]: audit 2024-07-15T06:48:43.750820+0000 mon.a (mon.0) 741 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:44.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:43 smithi089 bash[21793]: audit 2024-07-15T06:48:43.757692+0000 mon.a (mon.0) 742 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:44.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:43 smithi089 bash[31372]: cluster 2024-07-15T06:48:42.120327+0000 mgr.y (mgr.24272) 52 : cluster [DBG] pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:48:44.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:43 smithi089 bash[31372]: audit 2024-07-15T06:48:43.750820+0000 mon.a (mon.0) 741 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:44.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:43 smithi089 bash[31372]: audit 2024-07-15T06:48:43.757692+0000 mon.a (mon.0) 742 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:44.390 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:44 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:44.142295465Z level=info msg="Executing migration" id="drop index UQE_api_key_key - v1" 2024-07-15T06:48:44.390 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:44 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:44.167524319Z level=info msg="Executing migration" id="drop index UQE_api_key_account_id_name - v1" 2024-07-15T06:48:44.390 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:44 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:44.217853669Z level=info msg="Executing migration" id="Rename table api_key to api_key_v1 - v1" 2024-07-15T06:48:44.390 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:44 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:44.243180699Z level=info msg="Executing migration" id="create api_key table v2" 2024-07-15T06:48:44.390 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:44 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:44.285220756Z level=info msg="Executing migration" id="create index IDX_api_key_org_id - v2" 2024-07-15T06:48:44.390 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:44 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:44.336024294Z level=info msg="Executing migration" id="create index UQE_api_key_key - v2" 2024-07-15T06:48:44.390 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:44 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:44.352895455Z level=info msg="Executing migration" id="create index UQE_api_key_org_id_name - v2" 2024-07-15T06:48:44.390 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:44 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:44.394882947Z level=info msg="Executing migration" id="copy api_key v1 to v2" 2024-07-15T06:48:44.676 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:44 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:44.436873977Z level=info msg="Executing migration" id="Drop old table api_key_v1" 2024-07-15T06:48:44.676 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:44 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:44.462220941Z level=info msg="Executing migration" id="Update api_key table charset" 2024-07-15T06:48:44.676 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:44 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:44.487508666Z level=info msg="Executing migration" id="Add expires to api_key table" 2024-07-15T06:48:44.676 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:44 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:44.529628743Z level=info msg="Executing migration" id="Add service account foreign key" 2024-07-15T06:48:44.676 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:44 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:44.57174491Z level=info msg="Executing migration" id="set service account foreign key to nil if 0" 2024-07-15T06:48:44.676 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:44 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:44.597051842Z level=info msg="Executing migration" id="Add last_used_at to api_key table" 2024-07-15T06:48:44.677 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:44 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:44.639032438Z level=info msg="Executing migration" id="Add is_revoked column to api_key table" 2024-07-15T06:48:44.928 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:44 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:44.680931598Z level=info msg="Executing migration" id="create dashboard_snapshot table v4" 2024-07-15T06:48:44.928 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:44 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:44.706276107Z level=info msg="Executing migration" id="drop table dashboard_snapshot_v4 #1" 2024-07-15T06:48:44.929 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:44 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:44.748190026Z level=info msg="Executing migration" id="create dashboard_snapshot table v5 #2" 2024-07-15T06:48:44.929 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:44 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:44.79870197Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_key - v5" 2024-07-15T06:48:44.929 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:44 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:44.865995342Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_delete_key - v5" 2024-07-15T06:48:44.929 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:44 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:44.907875166Z level=info msg="Executing migration" id="create index IDX_dashboard_snapshot_user_id - v5" 2024-07-15T06:48:44.929 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:44 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:44.933320467Z level=info msg="Executing migration" id="alter dashboard_snapshot to mediumtext v2" 2024-07-15T06:48:45.188 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:44 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:44.958540884Z level=info msg="Executing migration" id="Update dashboard_snapshot table charset" 2024-07-15T06:48:45.189 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:44 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:44.992073862Z level=info msg="Executing migration" id="Add column external_delete_url to dashboard_snapshots table" 2024-07-15T06:48:45.189 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:45 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:45.025710503Z level=info msg="Executing migration" id="Add encrypted dashboard json column" 2024-07-15T06:48:45.189 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:45 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:45.059315102Z level=info msg="Executing migration" id="Change dashboard_encrypted column to MEDIUMBLOB" 2024-07-15T06:48:45.189 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:45 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:45.117875772Z level=info msg="Executing migration" id="create quota table v1" 2024-07-15T06:48:45.189 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:45 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:45.16847747Z level=info msg="Executing migration" id="create index UQE_quota_org_id_user_id_target - v1" 2024-07-15T06:48:45.441 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:45 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:45.218821235Z level=info msg="Executing migration" id="Update quota table charset" 2024-07-15T06:48:45.441 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:45 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:45.269413359Z level=info msg="Executing migration" id="create plugin_setting table" 2024-07-15T06:48:45.441 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:45 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:45.294788657Z level=info msg="Executing migration" id="create index UQE_plugin_setting_org_id_plugin_id - v1" 2024-07-15T06:48:45.441 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:45 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:45.336692508Z level=info msg="Executing migration" id="Add column plugin_version to plugin_settings" 2024-07-15T06:48:45.441 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:45 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:45.395402293Z level=info msg="Executing migration" id="Update plugin_setting table charset" 2024-07-15T06:48:45.441 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:45 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:45.420665979Z level=info msg="Executing migration" id="create session table" 2024-07-15T06:48:45.442 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:45 smithi134 bash[24068]: cluster 2024-07-15T06:48:44.120888+0000 mgr.y (mgr.24272) 53 : cluster [DBG] pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:48:45.442 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:45 smithi134 bash[24068]: audit 2024-07-15T06:48:44.194598+0000 mon.a (mon.0) 743 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:45.442 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:45 smithi134 bash[24068]: audit 2024-07-15T06:48:44.197212+0000 mon.a (mon.0) 744 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:48:45.442 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:45 smithi134 bash[24068]: audit 2024-07-15T06:48:44.745342+0000 mon.a (mon.0) 745 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:45.442 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:45 smithi134 bash[24068]: audit 2024-07-15T06:48:44.755233+0000 mon.a (mon.0) 746 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:45.442 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:45 smithi134 bash[24068]: audit 2024-07-15T06:48:44.757486+0000 mon.a (mon.0) 747 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:48:45.442 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:45 smithi134 bash[24068]: audit 2024-07-15T06:48:44.759204+0000 mon.a (mon.0) 748 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:48:45.442 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:45 smithi134 bash[24068]: audit 2024-07-15T06:48:44.769310+0000 mon.a (mon.0) 749 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:45.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:45 smithi089 bash[21793]: cluster 2024-07-15T06:48:44.120888+0000 mgr.y (mgr.24272) 53 : cluster [DBG] pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:48:45.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:45 smithi089 bash[21793]: audit 2024-07-15T06:48:44.194598+0000 mon.a (mon.0) 743 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:45.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:45 smithi089 bash[21793]: audit 2024-07-15T06:48:44.197212+0000 mon.a (mon.0) 744 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:48:45.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:45 smithi089 bash[21793]: audit 2024-07-15T06:48:44.745342+0000 mon.a (mon.0) 745 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:45.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:45 smithi089 bash[21793]: audit 2024-07-15T06:48:44.755233+0000 mon.a (mon.0) 746 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:45.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:45 smithi089 bash[21793]: audit 2024-07-15T06:48:44.757486+0000 mon.a (mon.0) 747 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:48:45.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:45 smithi089 bash[21793]: audit 2024-07-15T06:48:44.759204+0000 mon.a (mon.0) 748 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:48:45.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:45 smithi089 bash[21793]: audit 2024-07-15T06:48:44.769310+0000 mon.a (mon.0) 749 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:45.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:45 smithi089 bash[31372]: cluster 2024-07-15T06:48:44.120888+0000 mgr.y (mgr.24272) 53 : cluster [DBG] pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:48:45.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:45 smithi089 bash[31372]: audit 2024-07-15T06:48:44.194598+0000 mon.a (mon.0) 743 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:45.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:45 smithi089 bash[31372]: audit 2024-07-15T06:48:44.197212+0000 mon.a (mon.0) 744 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:48:45.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:45 smithi089 bash[31372]: audit 2024-07-15T06:48:44.745342+0000 mon.a (mon.0) 745 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:45.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:45 smithi089 bash[31372]: audit 2024-07-15T06:48:44.755233+0000 mon.a (mon.0) 746 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:45.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:45 smithi089 bash[31372]: audit 2024-07-15T06:48:44.757486+0000 mon.a (mon.0) 747 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:48:45.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:45 smithi089 bash[31372]: audit 2024-07-15T06:48:44.759204+0000 mon.a (mon.0) 748 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:48:45.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:45 smithi089 bash[31372]: audit 2024-07-15T06:48:44.769310+0000 mon.a (mon.0) 749 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:45.727 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:45 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:45.44597206Z level=info msg="Executing migration" id="Drop old table playlist table" 2024-07-15T06:48:45.727 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:45 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:45.487922032Z level=info msg="Executing migration" id="Drop old table playlist_item table" 2024-07-15T06:48:45.727 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:45 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:45.513137325Z level=info msg="Executing migration" id="create playlist table v2" 2024-07-15T06:48:45.727 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:45 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:45.538363075Z level=info msg="Executing migration" id="create playlist item table v2" 2024-07-15T06:48:45.727 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:45 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:45.563588862Z level=info msg="Executing migration" id="Update playlist table charset" 2024-07-15T06:48:45.727 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:45 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:45.58876377Z level=info msg="Executing migration" id="Update playlist_item table charset" 2024-07-15T06:48:45.727 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:45 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:45.639034007Z level=info msg="Executing migration" id="drop preferences table v2" 2024-07-15T06:48:45.727 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:45 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:45.66432239Z level=info msg="Executing migration" id="drop preferences table v3" 2024-07-15T06:48:45.727 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:45 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:45.689652901Z level=info msg="Executing migration" id="create preferences table v3" 2024-07-15T06:48:45.727 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:45 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:45.731559953Z level=info msg="Executing migration" id="Update preferences table charset" 2024-07-15T06:48:45.996 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:45 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:45.756800195Z level=info msg="Executing migration" id="Add column team_id in preferences" 2024-07-15T06:48:45.996 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:45 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:45.781942998Z level=info msg="Executing migration" id="Update team_id column values in preferences" 2024-07-15T06:48:45.996 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:45 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:45.807206552Z level=info msg="Executing migration" id="Add column week_start in preferences" 2024-07-15T06:48:45.996 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:45 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:45.832519442Z level=info msg="Executing migration" id="Add column preferences.json_data" 2024-07-15T06:48:45.996 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:45 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:45.874462756Z level=info msg="Executing migration" id="alter preferences.json_data to mediumtext v1" 2024-07-15T06:48:45.997 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:45 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:45.899741682Z level=info msg="Executing migration" id="Add preferences index org_id" 2024-07-15T06:48:45.997 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:45 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:45.92495279Z level=info msg="Executing migration" id="Add preferences index user_id" 2024-07-15T06:48:45.997 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:45 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:45.950326696Z level=info msg="Executing migration" id="create alert table v1" 2024-07-15T06:48:45.997 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:45 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:45.975520317Z level=info msg="Executing migration" id="add index alert org_id & id " 2024-07-15T06:48:45.997 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:46 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:46.000685985Z level=info msg="Executing migration" id="add index alert state" 2024-07-15T06:48:46.273 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:46 smithi134 bash[24068]: cephadm 2024-07-15T06:48:44.825222+0000 mgr.y (mgr.24272) 54 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2024-07-15T06:48:46.273 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:46 smithi134 bash[24068]: cephadm 2024-07-15T06:48:44.829810+0000 mgr.y (mgr.24272) 55 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi089 2024-07-15T06:48:46.274 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:46 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:46.025954429Z level=info msg="Executing migration" id="add index alert dashboard_id" 2024-07-15T06:48:46.274 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:46 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:46.051254712Z level=info msg="Executing migration" id="Create alert_rule_tag table v1" 2024-07-15T06:48:46.274 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:46 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:46.076439977Z level=info msg="Executing migration" id="Add unique index alert_rule_tag.alert_id_tag_id" 2024-07-15T06:48:46.274 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:46 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:46.101685535Z level=info msg="Executing migration" id="drop index UQE_alert_rule_tag_alert_id_tag_id - v1" 2024-07-15T06:48:46.274 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:46 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:46.143589148Z level=info msg="Executing migration" id="Rename table alert_rule_tag to alert_rule_tag_v1 - v1" 2024-07-15T06:48:46.274 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:46 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:46.168823582Z level=info msg="Executing migration" id="Create alert_rule_tag table v2" 2024-07-15T06:48:46.274 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:46 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:46.210852271Z level=info msg="Executing migration" id="create index UQE_alert_rule_tag_alert_id_tag_id - Add unique index alert_rule_tag.alert_id_tag_id V2" 2024-07-15T06:48:46.274 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:46 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:46.236345153Z level=info msg="Executing migration" id="copy alert_rule_tag v1 to v2" 2024-07-15T06:48:46.274 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:46 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:46.278287486Z level=info msg="Executing migration" id="drop table alert_rule_tag_v1" 2024-07-15T06:48:46.350 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-07-15T06:48:46.505 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:46 smithi089 bash[21793]: cephadm 2024-07-15T06:48:44.825222+0000 mgr.y (mgr.24272) 54 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2024-07-15T06:48:46.505 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:46 smithi089 bash[21793]: cephadm 2024-07-15T06:48:44.829810+0000 mgr.y (mgr.24272) 55 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi089 2024-07-15T06:48:46.506 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:46 smithi089 bash[31372]: cephadm 2024-07-15T06:48:44.825222+0000 mgr.y (mgr.24272) 54 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2024-07-15T06:48:46.506 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:46 smithi089 bash[31372]: cephadm 2024-07-15T06:48:44.829810+0000 mgr.y (mgr.24272) 55 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi089 2024-07-15T06:48:46.534 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:46 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:46.320328617Z level=info msg="Executing migration" id="create alert_notification table v1" 2024-07-15T06:48:46.534 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:46 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:46.345568791Z level=info msg="Executing migration" id="Add column is_default" 2024-07-15T06:48:46.534 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:46 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:46.387471868Z level=info msg="Executing migration" id="Add column frequency" 2024-07-15T06:48:46.534 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:46 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:46.412732882Z level=info msg="Executing migration" id="Add column send_reminder" 2024-07-15T06:48:46.535 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:46 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:46.437895884Z level=info msg="Executing migration" id="Add column disable_resolve_message" 2024-07-15T06:48:46.535 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:46 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:46.463181764Z level=info msg="Executing migration" id="add index alert_notification org_id & name" 2024-07-15T06:48:46.535 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:46 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:46.488447964Z level=info msg="Executing migration" id="Update alert table charset" 2024-07-15T06:48:46.535 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:46 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:46.513680918Z level=info msg="Executing migration" id="Update alert_notification table charset" 2024-07-15T06:48:46.535 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:46 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:46.538940929Z level=info msg="Executing migration" id="create notification_journal table v1" 2024-07-15T06:48:46.805 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:46 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:46.564179854Z level=info msg="Executing migration" id="add index notification_journal org_id & alert_id & notifier_id" 2024-07-15T06:48:46.805 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:46 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:46.589424494Z level=info msg="Executing migration" id="drop alert_notification_journal" 2024-07-15T06:48:46.805 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:46 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:46.614656032Z level=info msg="Executing migration" id="create alert_notification_state table v1" 2024-07-15T06:48:46.805 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:46 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:46.656554372Z level=info msg="Executing migration" id="add index alert_notification_state org_id & alert_id & notifier_id" 2024-07-15T06:48:46.805 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:46 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:46.673527328Z level=info msg="Executing migration" id="Add for to alert table" 2024-07-15T06:48:46.805 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:46 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:46.715467892Z level=info msg="Executing migration" id="Add column uid in alert_notification" 2024-07-15T06:48:46.806 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:46 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:46.757419568Z level=info msg="Executing migration" id="Update uid column values in alert_notification" 2024-07-15T06:48:46.806 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:46 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:46.782701978Z level=info msg="Executing migration" id="Add unique index alert_notification_org_id_uid" 2024-07-15T06:48:47.097 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:48:46 smithi134 bash[25019]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:48:46] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-07-15T06:48:47.097 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:46 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:46.824721549Z level=info msg="Executing migration" id="Remove unique index org_id_name" 2024-07-15T06:48:47.097 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:46 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:46.84993109Z level=info msg="Executing migration" id="Add column secure_settings in alert_notification" 2024-07-15T06:48:47.097 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:46 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:46.891879833Z level=info msg="Executing migration" id="alter alert.settings to mediumtext" 2024-07-15T06:48:47.097 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:46 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:46.917149563Z level=info msg="Executing migration" id="Add non-unique index alert_notification_state_alert_id" 2024-07-15T06:48:47.098 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:46 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:46.942364485Z level=info msg="Executing migration" id="Add non-unique index alert_rule_tag_alert_id" 2024-07-15T06:48:47.098 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:46 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:46.967573666Z level=info msg="Executing migration" id="Drop old annotation table v4" 2024-07-15T06:48:47.098 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:47 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:47.009714045Z level=info msg="Executing migration" id="create annotation table v5" 2024-07-15T06:48:47.098 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:47 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:47.034967Z level=info msg="Executing migration" id="add index annotation 0 v3" 2024-07-15T06:48:47.098 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:47 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:47.060119584Z level=info msg="Executing migration" id="add index annotation 1 v3" 2024-07-15T06:48:47.098 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:47 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:47.102019665Z level=info msg="Executing migration" id="add index annotation 2 v3" 2024-07-15T06:48:47.350 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:47 smithi134 bash[24068]: cluster 2024-07-15T06:48:46.121390+0000 mgr.y (mgr.24272) 56 : cluster [DBG] pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:48:47.350 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:47 smithi134 bash[24068]: audit 2024-07-15T06:48:46.349760+0000 mon.c (mon.2) 19 : audit [DBG] from='client.? 172.21.15.89:0/3002786662' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-07-15T06:48:47.350 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:47 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:47.127471816Z level=info msg="Executing migration" id="add index annotation 3 v3" 2024-07-15T06:48:47.350 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:47 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:47.161284083Z level=info msg="Executing migration" id="add index annotation 4 v3" 2024-07-15T06:48:47.351 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:47 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:47.203278334Z level=info msg="Executing migration" id="Update annotation table charset" 2024-07-15T06:48:47.351 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:47 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:47.24533874Z level=info msg="Executing migration" id="Add column region_id to annotation table" 2024-07-15T06:48:47.351 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:47 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:47.287322329Z level=info msg="Executing migration" id="Drop category_id index" 2024-07-15T06:48:47.351 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:47 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:47.329386805Z level=info msg="Executing migration" id="Add column tags to annotation table" 2024-07-15T06:48:47.351 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:47 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:47.354690502Z level=info msg="Executing migration" id="Create annotation_tag table v2" 2024-07-15T06:48:47.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:47 smithi089 bash[21793]: cluster 2024-07-15T06:48:46.121390+0000 mgr.y (mgr.24272) 56 : cluster [DBG] pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:48:47.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:47 smithi089 bash[21793]: audit 2024-07-15T06:48:46.349760+0000 mon.c (mon.2) 19 : audit [DBG] from='client.? 172.21.15.89:0/3002786662' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-07-15T06:48:47.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:47 smithi089 bash[31372]: cluster 2024-07-15T06:48:46.121390+0000 mgr.y (mgr.24272) 56 : cluster [DBG] pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:48:47.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:47 smithi089 bash[31372]: audit 2024-07-15T06:48:46.349760+0000 mon.c (mon.2) 19 : audit [DBG] from='client.? 172.21.15.89:0/3002786662' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-07-15T06:48:47.635 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:47 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:47.396618067Z level=info msg="Executing migration" id="Add unique index annotation_tag.annotation_id_tag_id" 2024-07-15T06:48:47.636 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:47 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:47.421897741Z level=info msg="Executing migration" id="drop index UQE_annotation_tag_annotation_id_tag_id - v2" 2024-07-15T06:48:47.636 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:47 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:47.447087544Z level=info msg="Executing migration" id="Rename table annotation_tag to annotation_tag_v2 - v2" 2024-07-15T06:48:47.636 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:47 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:47.497384814Z level=info msg="Executing migration" id="Create annotation_tag table v3" 2024-07-15T06:48:47.636 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:47 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:47.522833815Z level=info msg="Executing migration" id="create index UQE_annotation_tag_annotation_id_tag_id - Add unique index annotation_tag.annotation_id_tag_id V3" 2024-07-15T06:48:47.636 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:47 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:47.548105114Z level=info msg="Executing migration" id="copy annotation_tag v2 to v3" 2024-07-15T06:48:47.636 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:47 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:47.573205775Z level=info msg="Executing migration" id="drop table annotation_tag_v2" 2024-07-15T06:48:47.636 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:47 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:47.59845692Z level=info msg="Executing migration" id="Update alert annotations and set TEXT to empty" 2024-07-15T06:48:47.637 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:47 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:47.640371868Z level=info msg="Executing migration" id="Add created time to annotation table" 2024-07-15T06:48:47.689 INFO:teuthology.orchestra.run.smithi089.stdout:{"epoch":20,"flags":0,"active_gid":24272,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6800","nonce":2527934197},{"type":"v1","addr":"172.21.15.89:6801","nonce":2527934197}]},"active_addr":"172.21.15.89:6801/2527934197","active_change":"2024-07-15T06:47:44.086667+0000","active_mgr_features":4540138320759226367,"available":true,"standbys":[{"gid":24310,"name":"x","mgr_features":4540138320759226367,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.89:8443/","prometheus":"http://172.21.15.89:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":59,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.89:0","nonce":1034002740}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.89:0","nonce":1249162602}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.89:0","nonce":3524047244}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.89:0","nonce":2281831659}]}]} 2024-07-15T06:48:47.693 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-07-15T06:48:47.693 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-07-15T06:48:47.693 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph osd dump --format=json 2024-07-15T06:48:47.888 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:47 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:47.665688085Z level=info msg="Executing migration" id="Add updated time to annotation table" 2024-07-15T06:48:47.888 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:47 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:47.707591753Z level=info msg="Executing migration" id="Add index for created in annotation table" 2024-07-15T06:48:47.888 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:47 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:47.732844662Z level=info msg="Executing migration" id="Add index for updated in annotation table" 2024-07-15T06:48:47.888 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:47 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:47.774834719Z level=info msg="Executing migration" id="Convert existing annotations from seconds to milliseconds" 2024-07-15T06:48:47.889 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:47 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:47.791641616Z level=info msg="Executing migration" id="Add epoch_end column" 2024-07-15T06:48:47.889 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:47 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:47.80857997Z level=info msg="Executing migration" id="Add index for epoch_end" 2024-07-15T06:48:47.889 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:47 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:47.825543285Z level=info msg="Executing migration" id="Make epoch_end the same as epoch" 2024-07-15T06:48:47.889 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:47 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:47.842408137Z level=info msg="Executing migration" id="Move region to single row" 2024-07-15T06:48:47.889 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:47 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:47.867665362Z level=info msg="Executing migration" id="Remove index org_id_epoch from annotation table" 2024-07-15T06:48:47.889 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:47 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:47.892981653Z level=info msg="Executing migration" id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2024-07-15T06:48:48.157 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:47 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:47.918102664Z level=info msg="Executing migration" id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2024-07-15T06:48:48.158 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:47 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:47.985210295Z level=info msg="Executing migration" id="Add index for org_id_epoch_end_epoch on annotation table" 2024-07-15T06:48:48.158 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:48 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:48.010432653Z level=info msg="Executing migration" id="Remove index org_id_epoch_epoch_end from annotation table" 2024-07-15T06:48:48.158 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:48 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:48.035680653Z level=info msg="Executing migration" id="Add index for alert_id on annotation table" 2024-07-15T06:48:48.158 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:48 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:48.060898283Z level=info msg="Executing migration" id="Increase tags column to length 4096" 2024-07-15T06:48:48.158 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:48 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:48.086421714Z level=info msg="Executing migration" id="create test_data table" 2024-07-15T06:48:48.158 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:48 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:48.111780651Z level=info msg="Executing migration" id="create dashboard_version table v1" 2024-07-15T06:48:48.158 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:48 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:48.136919552Z level=info msg="Executing migration" id="add index dashboard_version.dashboard_id" 2024-07-15T06:48:48.158 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:48 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:48.162025767Z level=info msg="Executing migration" id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2024-07-15T06:48:48.409 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:48 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:48.187383975Z level=info msg="Executing migration" id="Set dashboard version to 1 where 0" 2024-07-15T06:48:48.409 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:48 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:48.229289419Z level=info msg="Executing migration" id="save existing dashboard data in dashboard_version table v1" 2024-07-15T06:48:48.410 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:48 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:48.271224818Z level=info msg="Executing migration" id="alter dashboard_version.data to mediumtext v1" 2024-07-15T06:48:48.410 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:48 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:48.313257546Z level=info msg="Executing migration" id="create team table" 2024-07-15T06:48:48.410 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:48 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:48.338522982Z level=info msg="Executing migration" id="add index team.org_id" 2024-07-15T06:48:48.410 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:48 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:48.363735341Z level=info msg="Executing migration" id="add unique index team_org_id_name" 2024-07-15T06:48:48.410 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:48 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:48.389039909Z level=info msg="Executing migration" id="create team member table" 2024-07-15T06:48:48.410 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:48 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:48.414167981Z level=info msg="Executing migration" id="add index team_member.org_id" 2024-07-15T06:48:48.695 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:48 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:48.439493169Z level=info msg="Executing migration" id="add unique index team_member_org_id_team_id_user_id" 2024-07-15T06:48:48.695 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:48 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:48.464761245Z level=info msg="Executing migration" id="add index team_member.team_id" 2024-07-15T06:48:48.696 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:48 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:48.489998764Z level=info msg="Executing migration" id="Add column email to team table" 2024-07-15T06:48:48.696 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:48 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:48.515182175Z level=info msg="Executing migration" id="Add column external to team_member table" 2024-07-15T06:48:48.696 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:48 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:48.540385959Z level=info msg="Executing migration" id="Add column permission to team_member table" 2024-07-15T06:48:48.696 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:48 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:48.565709507Z level=info msg="Executing migration" id="create dashboard acl table" 2024-07-15T06:48:48.696 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:48 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:48.590955229Z level=info msg="Executing migration" id="add index dashboard_acl_dashboard_id" 2024-07-15T06:48:48.696 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:48 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:48.632906778Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_user_id" 2024-07-15T06:48:48.696 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:48 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:48.658150754Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_team_id" 2024-07-15T06:48:48.696 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:48 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:48.700092491Z level=info msg="Executing migration" id="add index dashboard_acl_user_id" 2024-07-15T06:48:48.965 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:48 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:48.725525233Z level=info msg="Executing migration" id="add index dashboard_acl_team_id" 2024-07-15T06:48:48.965 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:48 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:48.767359202Z level=info msg="Executing migration" id="add index dashboard_acl_org_id_role" 2024-07-15T06:48:48.965 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:48 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:48.792620181Z level=info msg="Executing migration" id="add index dashboard_permission" 2024-07-15T06:48:48.965 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:48 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:48.817872624Z level=info msg="Executing migration" id="save default acl rules in dashboard_acl table" 2024-07-15T06:48:48.965 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:48 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:48.843109143Z level=info msg="Executing migration" id="delete acl rules for deleted dashboards and folders" 2024-07-15T06:48:48.965 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:48 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:48.868419391Z level=info msg="Executing migration" id="create tag table" 2024-07-15T06:48:48.965 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:48 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:48.893602922Z level=info msg="Executing migration" id="add index tag.key_value" 2024-07-15T06:48:48.965 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:48 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:48.918832161Z level=info msg="Executing migration" id="create login attempt table" 2024-07-15T06:48:48.965 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:48 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:48.944075348Z level=info msg="Executing migration" id="add index login_attempt.username" 2024-07-15T06:48:48.965 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:48 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:48.969339293Z level=info msg="Executing migration" id="drop index IDX_login_attempt_username - v1" 2024-07-15T06:48:49.216 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:49 smithi134 bash[24068]: cluster 2024-07-15T06:48:48.121905+0000 mgr.y (mgr.24272) 57 : cluster [DBG] pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:48:49.217 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:48 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:48.994798342Z level=info msg="Executing migration" id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2024-07-15T06:48:49.217 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:49 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:49.036676917Z level=info msg="Executing migration" id="create login_attempt v2" 2024-07-15T06:48:49.217 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:49 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:49.061858205Z level=info msg="Executing migration" id="create index IDX_login_attempt_username - v2" 2024-07-15T06:48:49.217 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:49 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:49.103897228Z level=info msg="Executing migration" id="copy login_attempt v1 to v2" 2024-07-15T06:48:49.217 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:49 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:49.129078693Z level=info msg="Executing migration" id="drop login_attempt_tmp_qwerty" 2024-07-15T06:48:49.217 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:49 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:49.154376589Z level=info msg="Executing migration" id="create user auth table" 2024-07-15T06:48:49.217 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:49 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:49.221379664Z level=info msg="Executing migration" id="create index IDX_user_auth_auth_module_auth_id - v1" 2024-07-15T06:48:49.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:49 smithi089 bash[21793]: cluster 2024-07-15T06:48:48.121905+0000 mgr.y (mgr.24272) 57 : cluster [DBG] pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:48:49.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:49 smithi089 bash[31372]: cluster 2024-07-15T06:48:48.121905+0000 mgr.y (mgr.24272) 57 : cluster [DBG] pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:48:49.469 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:49 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:49.246644168Z level=info msg="Executing migration" id="alter user_auth.auth_id to length 190" 2024-07-15T06:48:49.469 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:49 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:49.288564293Z level=info msg="Executing migration" id="Add OAuth access token to user_auth" 2024-07-15T06:48:49.469 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:49 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:49.330500391Z level=info msg="Executing migration" id="Add OAuth refresh token to user_auth" 2024-07-15T06:48:49.469 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:49 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:49.355786053Z level=info msg="Executing migration" id="Add OAuth token type to user_auth" 2024-07-15T06:48:49.469 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:49 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:49.397814694Z level=info msg="Executing migration" id="Add OAuth expiry to user_auth" 2024-07-15T06:48:49.469 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:49 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:49.423102249Z level=info msg="Executing migration" id="Add index to user_id column in user_auth" 2024-07-15T06:48:49.469 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:49 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:49.44832736Z level=info msg="Executing migration" id="Add OAuth ID token to user_auth" 2024-07-15T06:48:49.469 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:49 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:49.473502346Z level=info msg="Executing migration" id="create server_lock table" 2024-07-15T06:48:49.721 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:49 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:49.498809218Z level=info msg="Executing migration" id="add index server_lock.operation_uid" 2024-07-15T06:48:49.721 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:49 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:49.524049442Z level=info msg="Executing migration" id="create user auth token table" 2024-07-15T06:48:49.721 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:49 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:49.549269512Z level=info msg="Executing migration" id="add unique index user_auth_token.auth_token" 2024-07-15T06:48:49.721 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:49 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:49.574504206Z level=info msg="Executing migration" id="add unique index user_auth_token.prev_auth_token" 2024-07-15T06:48:49.721 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:49 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:49.616442928Z level=info msg="Executing migration" id="add index user_auth_token.user_id" 2024-07-15T06:48:49.721 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:49 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:49.658387859Z level=info msg="Executing migration" id="Add revoked_at to the user auth token" 2024-07-15T06:48:49.721 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:49 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:49.70032718Z level=info msg="Executing migration" id="create cache_data table" 2024-07-15T06:48:49.721 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:49 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:49.725668999Z level=info msg="Executing migration" id="add unique index cache_data.cache_key" 2024-07-15T06:48:50.034 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:49 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:49.767559281Z level=info msg="Executing migration" id="create short_url table v1" 2024-07-15T06:48:50.034 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:49 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:49.809633446Z level=info msg="Executing migration" id="add index short_url.org_id-uid" 2024-07-15T06:48:50.034 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:49 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:49.834862047Z level=info msg="Executing migration" id="alter table short_url alter column created_by type to bigint" 2024-07-15T06:48:50.034 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:49 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:49.860023495Z level=info msg="Executing migration" id="delete alert_definition table" 2024-07-15T06:48:50.035 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:49 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:49.885277669Z level=info msg="Executing migration" id="recreate alert_definition table" 2024-07-15T06:48:50.035 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:49 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:49.927320525Z level=info msg="Executing migration" id="add index in alert_definition on org_id and title columns" 2024-07-15T06:48:50.035 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:50 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:50.039001169Z level=info msg="Executing migration" id="add index in alert_definition on org_id and uid columns" 2024-07-15T06:48:50.082 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 06:48:49 smithi089 systemd[1]: Stopping Ceph alertmanager.a for 75ba02bc-4274-11ef-bcad-c7b262605968... 2024-07-15T06:48:50.082 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 06:48:49 smithi089 bash[56397]: ts=2024-07-15T06:48:49.832Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-07-15T06:48:50.299 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:50 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:50.094662573Z level=info msg="Executing migration" id="alter alert_definition table data column to mediumtext in mysql" 2024-07-15T06:48:50.299 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:50 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:50.193720151Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and title columns" 2024-07-15T06:48:50.332 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 06:48:50 smithi089 bash[58007]: ceph-75ba02bc-4274-11ef-bcad-c7b262605968-alertmanager-a 2024-07-15T06:48:50.551 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:50 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:50.303921217Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and uid columns" 2024-07-15T06:48:50.551 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:50 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:50.36260135Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and title columns" 2024-07-15T06:48:50.551 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:50 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:50.387921407Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and uid columns" 2024-07-15T06:48:50.552 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:50 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:50.413443226Z level=info msg="Executing migration" id="Add column paused in alert_definition" 2024-07-15T06:48:50.552 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:50 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:50.455130988Z level=info msg="Executing migration" id="drop alert_definition table" 2024-07-15T06:48:50.552 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:50 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:50.480454411Z level=info msg="Executing migration" id="delete alert_definition_version table" 2024-07-15T06:48:50.552 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:50 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:50.505721398Z level=info msg="Executing migration" id="recreate alert_definition_version table" 2024-07-15T06:48:50.552 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:50 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:50.530935346Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_id and version columns" 2024-07-15T06:48:50.552 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:50 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:50.556157812Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_uid and version columns" 2024-07-15T06:48:50.805 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:50 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:50.598015129Z level=info msg="Executing migration" id="alter alert_definition_version table data column to mediumtext in mysql" 2024-07-15T06:48:50.805 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:50 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:50.64007706Z level=info msg="Executing migration" id="drop alert_definition_version table" 2024-07-15T06:48:50.805 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:50 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:50.682062769Z level=info msg="Executing migration" id="create alert_instance table" 2024-07-15T06:48:50.805 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:50 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:50.72407627Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, def_uid and current_state columns" 2024-07-15T06:48:50.805 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:50 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:50.749475422Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, current_state columns" 2024-07-15T06:48:50.805 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:50 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:50.774665307Z level=info msg="Executing migration" id="add column current_state_end to alert_instance" 2024-07-15T06:48:50.869 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 06:48:50 smithi089 bash[58068]: Error response from daemon: No such container: ceph-75ba02bc-4274-11ef-bcad-c7b262605968-alertmanager-a 2024-07-15T06:48:50.869 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 06:48:50 smithi089 systemd[1]: ceph-75ba02bc-4274-11ef-bcad-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-07-15T06:48:50.869 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 06:48:50 smithi089 systemd[1]: Stopped Ceph alertmanager.a for 75ba02bc-4274-11ef-bcad-c7b262605968. 2024-07-15T06:48:50.869 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 06:48:50 smithi089 systemd[1]: Started Ceph alertmanager.a for 75ba02bc-4274-11ef-bcad-c7b262605968. 2024-07-15T06:48:51.142 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:50 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:50.825471228Z level=info msg="Executing migration" id="remove index def_org_id, def_uid, current_state on alert_instance" 2024-07-15T06:48:51.143 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:50 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:50.892336571Z level=info msg="Executing migration" id="remove index def_org_id, current_state on alert_instance" 2024-07-15T06:48:51.143 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:51 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:51.147489944Z level=info msg="Executing migration" id="rename def_org_id to rule_org_id in alert_instance" 2024-07-15T06:48:51.805 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:51 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:51.511544278Z level=info msg="Executing migration" id="rename def_uid to rule_uid in alert_instance" 2024-07-15T06:48:51.805 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:51 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:51.553982122Z level=info msg="Executing migration" id="add index rule_org_id, rule_uid, current_state on alert_instance" 2024-07-15T06:48:51.805 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:51 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:51.629990954Z level=info msg="Executing migration" id="add index rule_org_id, current_state on alert_instance" 2024-07-15T06:48:51.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:51 smithi134 bash[24068]: cluster 2024-07-15T06:48:50.122395+0000 mgr.y (mgr.24272) 58 : cluster [DBG] pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:48:51.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:51 smithi134 bash[24068]: audit 2024-07-15T06:48:50.572925+0000 mon.a (mon.0) 750 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:51.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:51 smithi134 bash[24068]: audit 2024-07-15T06:48:50.578448+0000 mon.a (mon.0) 751 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:51.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:51 smithi134 bash[24068]: cephadm 2024-07-15T06:48:50.579817+0000 mgr.y (mgr.24272) 59 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2024-07-15T06:48:51.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:51 smithi134 bash[24068]: cephadm 2024-07-15T06:48:50.589011+0000 mgr.y (mgr.24272) 60 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi134 2024-07-15T06:48:51.819 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:51 smithi089 bash[21793]: cluster 2024-07-15T06:48:50.122395+0000 mgr.y (mgr.24272) 58 : cluster [DBG] pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:48:51.820 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:51 smithi089 bash[21793]: audit 2024-07-15T06:48:50.572925+0000 mon.a (mon.0) 750 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:51.820 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:51 smithi089 bash[21793]: audit 2024-07-15T06:48:50.578448+0000 mon.a (mon.0) 751 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:51.820 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:51 smithi089 bash[21793]: cephadm 2024-07-15T06:48:50.579817+0000 mgr.y (mgr.24272) 59 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2024-07-15T06:48:51.820 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:51 smithi089 bash[21793]: cephadm 2024-07-15T06:48:50.589011+0000 mgr.y (mgr.24272) 60 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi134 2024-07-15T06:48:51.820 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:51 smithi089 bash[31372]: cluster 2024-07-15T06:48:50.122395+0000 mgr.y (mgr.24272) 58 : cluster [DBG] pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:48:51.820 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:51 smithi089 bash[31372]: audit 2024-07-15T06:48:50.572925+0000 mon.a (mon.0) 750 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:51.820 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:51 smithi089 bash[31372]: audit 2024-07-15T06:48:50.578448+0000 mon.a (mon.0) 751 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:51.820 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:51 smithi089 bash[31372]: cephadm 2024-07-15T06:48:50.579817+0000 mgr.y (mgr.24272) 59 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2024-07-15T06:48:51.820 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:51 smithi089 bash[31372]: cephadm 2024-07-15T06:48:50.589011+0000 mgr.y (mgr.24272) 60 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi134 2024-07-15T06:48:51.820 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 06:48:51 smithi089 bash[58097]: ts=2024-07-15T06:48:51.788Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-07-15T06:48:51.821 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 06:48:51 smithi089 bash[58097]: ts=2024-07-15T06:48:51.788Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-07-15T06:48:51.821 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 06:48:51 smithi089 bash[58097]: ts=2024-07-15T06:48:51.790Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.89 port=9094 2024-07-15T06:48:51.821 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 06:48:51 smithi089 bash[58097]: ts=2024-07-15T06:48:51.791Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-07-15T06:48:51.821 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 06:48:51 smithi089 bash[58097]: ts=2024-07-15T06:48:51.821Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-07-15T06:48:51.821 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 06:48:51 smithi089 bash[58097]: ts=2024-07-15T06:48:51.822Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-07-15T06:48:51.821 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 06:48:51 smithi089 bash[58097]: ts=2024-07-15T06:48:51.825Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-07-15T06:48:51.821 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 06:48:51 smithi089 bash[58097]: ts=2024-07-15T06:48:51.825Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-07-15T06:48:52.253 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:51 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:51.941451411Z level=info msg="Executing migration" id="add current_reason column related to current_state" 2024-07-15T06:48:52.253 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:52 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:52.073207081Z level=info msg="Executing migration" id="create alert_rule table" 2024-07-15T06:48:52.253 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:52 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:52.173883284Z level=info msg="Executing migration" id="add index in alert_rule on org_id and title columns" 2024-07-15T06:48:52.253 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:52 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:52.257731733Z level=info msg="Executing migration" id="add index in alert_rule on org_id and uid columns" 2024-07-15T06:48:52.555 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:52 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:52.316385952Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespace_uid, group_uid columns" 2024-07-15T06:48:52.555 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:52 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:52.408559043Z level=info msg="Executing migration" id="alter alert_rule table data column to mediumtext in mysql" 2024-07-15T06:48:52.555 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:52 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:52.487409929Z level=info msg="Executing migration" id="add column for to alert_rule" 2024-07-15T06:48:52.843 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:52 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:52.58165985Z level=info msg="Executing migration" id="add column annotations to alert_rule" 2024-07-15T06:48:52.843 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:52 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:52.647030656Z level=info msg="Executing migration" id="add column labels to alert_rule" 2024-07-15T06:48:52.843 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:52 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:52.705557425Z level=info msg="Executing migration" id="remove unique index from alert_rule on org_id, title columns" 2024-07-15T06:48:52.843 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:52 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:52.790918426Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespase_uid and title columns" 2024-07-15T06:48:52.843 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:52 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:52.848326783Z level=info msg="Executing migration" id="add dashboard_uid column to alert_rule" 2024-07-15T06:48:53.095 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:52 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:52.937826121Z level=info msg="Executing migration" id="add panel_id column to alert_rule" 2024-07-15T06:48:53.095 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:53 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:53.044200451Z level=info msg="Executing migration" id="add index in alert_rule on org_id, dashboard_uid and panel_id columns" 2024-07-15T06:48:53.493 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:53 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:53.245816296Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule" 2024-07-15T06:48:53.779 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:53 smithi134 bash[24068]: cluster 2024-07-15T06:48:52.123020+0000 mgr.y (mgr.24272) 61 : cluster [DBG] pgmap v41: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:48:53.779 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:53 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:53.497894909Z level=info msg="Executing migration" id="add is_paused column to alert_rule table" 2024-07-15T06:48:53.779 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:53 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:53.640317315Z level=info msg="Executing migration" id="fix is_paused column for alert_rule table" 2024-07-15T06:48:53.779 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:53 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:53.715908098Z level=info msg="Executing migration" id="create alert_rule_version table" 2024-07-15T06:48:53.869 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:53 smithi089 bash[21793]: cluster 2024-07-15T06:48:52.123020+0000 mgr.y (mgr.24272) 61 : cluster [DBG] pgmap v41: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:48:53.869 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:53 smithi089 bash[31372]: cluster 2024-07-15T06:48:52.123020+0000 mgr.y (mgr.24272) 61 : cluster [DBG] pgmap v41: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:48:53.870 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 06:48:53 smithi089 bash[58097]: ts=2024-07-15T06:48:53.792Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.00061054s 2024-07-15T06:48:54.055 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:53 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:53.783621903Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_uid and version columns" 2024-07-15T06:48:54.055 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:53 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:53.825615331Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_namespace_uid and rule_group columns" 2024-07-15T06:48:54.055 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:53 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:53.884565735Z level=info msg="Executing migration" id="alter alert_rule_version table data column to mediumtext in mysql" 2024-07-15T06:48:54.064 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T06:48:54.383 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:54 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:54.089295049Z level=info msg="Executing migration" id="add column for to alert_rule_version" 2024-07-15T06:48:54.383 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:54 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:54.240437507Z level=info msg="Executing migration" id="add column annotations to alert_rule_version" 2024-07-15T06:48:54.383 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:54 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:54.387570004Z level=info msg="Executing migration" id="add column labels to alert_rule_version" 2024-07-15T06:48:54.714 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:54 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:54.463600288Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule_version" 2024-07-15T06:48:55.044 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:54 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:54.719460882Z level=info msg="Executing migration" id="add is_paused column to alert_rule_versions table" 2024-07-15T06:48:55.044 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:54 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:54.890010625Z level=info msg="Executing migration" id="fix is_paused column for alert_rule_version table" 2024-07-15T06:48:55.044 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:54 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:54.952106561Z level=info msg="Executing migration" id=create_alert_configuration_table 2024-07-15T06:48:55.045 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:55 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:55.049289058Z level=info msg="Executing migration" id="Add column default in alert_configuration" 2024-07-15T06:48:55.305 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:55 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:55.141771605Z level=info msg="Executing migration" id="alert alert_configuration alertmanager_configuration column from TEXT to MEDIUMTEXT if mysql" 2024-07-15T06:48:55.305 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:55 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:55.191886914Z level=info msg="Executing migration" id="add column org_id in alert_configuration" 2024-07-15T06:48:55.305 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:55 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:55.275801242Z level=info msg="Executing migration" id="add index in alert_configuration table on org_id column" 2024-07-15T06:48:55.658 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:55 smithi134 bash[24068]: cluster 2024-07-15T06:48:54.123634+0000 mgr.y (mgr.24272) 62 : cluster [DBG] pgmap v42: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:48:55.658 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:55 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:55.343128244Z level=info msg="Executing migration" id="add configuration_hash column to alert_configuration" 2024-07-15T06:48:55.658 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:55 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:55.385119501Z level=info msg="Executing migration" id=create_ngalert_configuration_table 2024-07-15T06:48:55.659 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:55 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:55.410393321Z level=info msg="Executing migration" id="add index in ngalert_configuration on org_id column" 2024-07-15T06:48:55.659 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:55 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:55.45254963Z level=info msg="Executing migration" id="add column send_alerts_to in ngalert_configuration" 2024-07-15T06:48:55.659 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:55 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:55.51177044Z level=info msg="Executing migration" id="create provenance_type table" 2024-07-15T06:48:55.659 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:55 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:55.570819046Z level=info msg="Executing migration" id="add index to uniquify (record_key, record_type, org_id) columns" 2024-07-15T06:48:55.659 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:55 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:55.663071938Z level=info msg="Executing migration" id="create alert_image table" 2024-07-15T06:48:55.869 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:55 smithi089 bash[31372]: cluster 2024-07-15T06:48:54.123634+0000 mgr.y (mgr.24272) 62 : cluster [DBG] pgmap v42: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:48:55.869 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:55 smithi089 bash[21793]: cluster 2024-07-15T06:48:54.123634+0000 mgr.y (mgr.24272) 62 : cluster [DBG] pgmap v42: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:48:55.942 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:55 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:55.788345934Z level=info msg="Executing migration" id="add unique index on token to alert_image table" 2024-07-15T06:48:56.305 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:55 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:55.947404706Z level=info msg="Executing migration" id="support longer URLs in alert_image table" 2024-07-15T06:48:56.305 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:56 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:56.195318188Z level=info msg="Executing migration" id=create_alert_configuration_history_table 2024-07-15T06:48:56.697 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:48:56 smithi134 systemd[1]: Stopping Ceph prometheus.a for 75ba02bc-4274-11ef-bcad-c7b262605968... 2024-07-15T06:48:56.697 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:56 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:56.382251842Z level=info msg="Executing migration" id="drop non-unique orgID index on alert_configuration" 2024-07-15T06:48:56.697 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:56 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:56.457851431Z level=info msg="Executing migration" id="drop unique orgID index on alert_configuration if exists" 2024-07-15T06:48:56.697 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:56 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:56.458500898Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop unique orgID index on alert_configuration if exists" 2024-07-15T06:48:56.697 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:56 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:56.533856394Z level=info msg="Executing migration" id="extract alertmanager configuration history to separate table" 2024-07-15T06:48:56.698 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:56 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:56.575595917Z level=info msg="Executing migration" id="add unique index on orgID to alert_configuration" 2024-07-15T06:48:56.698 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:56 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:56.624350745Z level=info msg="Executing migration" id="clear migration entry \"remove unified alerting data\"" 2024-07-15T06:48:56.698 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:56 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:56.624652297Z level=info msg="Executing migration" id="move dashboard alerts to unified alerting" 2024-07-15T06:48:56.698 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:56 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:56.62488285Z level=info msg="alerts found to migrate" alerts=0 2024-07-15T06:48:56.698 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:56 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:56.701886276Z level=info msg="Executing migration" id="create library_element table v1" 2024-07-15T06:48:57.024 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:56 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:56.727300756Z level=info msg="Executing migration" id="add index library_element org_id-folder_id-name-kind" 2024-07-15T06:48:57.025 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:56 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:56.769121522Z level=info msg="Executing migration" id="create library_element_connection table v1" 2024-07-15T06:48:57.025 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:56 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:56.822397364Z level=info msg="Executing migration" id="add index library_element_connection element_id-kind-connection_id" 2024-07-15T06:48:57.025 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:56 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:56.920438658Z level=info msg="Executing migration" id="add unique index library_element org_id_uid" 2024-07-15T06:48:57.025 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:57 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:57.029174129Z level=info msg="Executing migration" id="increase max description length to 2048" 2024-07-15T06:48:57.025 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:48:56 smithi134 bash[48372]: ts=2024-07-15T06:48:56.731Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-07-15T06:48:57.025 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:48:56 smithi134 bash[48372]: ts=2024-07-15T06:48:56.731Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-07-15T06:48:57.025 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:48:56 smithi134 bash[48372]: ts=2024-07-15T06:48:56.731Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-07-15T06:48:57.025 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:48:56 smithi134 bash[48372]: ts=2024-07-15T06:48:56.731Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-07-15T06:48:57.025 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:48:56 smithi134 bash[48372]: ts=2024-07-15T06:48:56.731Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-07-15T06:48:57.025 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:48:56 smithi134 bash[48372]: ts=2024-07-15T06:48:56.731Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-07-15T06:48:57.026 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:48:56 smithi134 bash[48372]: ts=2024-07-15T06:48:56.731Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-07-15T06:48:57.026 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:48:56 smithi134 bash[48372]: ts=2024-07-15T06:48:56.731Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-07-15T06:48:57.026 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:48:56 smithi134 bash[48372]: ts=2024-07-15T06:48:56.732Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-07-15T06:48:57.026 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:48:56 smithi134 bash[48372]: ts=2024-07-15T06:48:56.732Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-07-15T06:48:57.026 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:48:56 smithi134 bash[48372]: ts=2024-07-15T06:48:56.732Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-07-15T06:48:57.026 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:48:56 smithi134 bash[48372]: ts=2024-07-15T06:48:56.732Z caller=main.go:1147 level=info msg="See you next time!" 2024-07-15T06:48:57.305 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:57 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:57.213491593Z level=info msg="Executing migration" id="clone move dashboard alerts to unified alerting" 2024-07-15T06:48:57.458 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-07-15T06:48:57.458 INFO:teuthology.orchestra.run.smithi089.stdout:{"epoch":63,"fsid":"75ba02bc-4274-11ef-bcad-c7b262605968","created":"2024-07-15T06:37:09.634877+0000","modified":"2024-07-15T06:48:36.696104+0000","last_up_change":"2024-07-15T06:46:45.306680+0000","last_in_change":"2024-07-15T06:46:19.019927+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-07-15T06:42:21.444347+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2024-07-15T06:48:27.276344+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"79357114-251f-489d-bef5-f6ec37002212","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6802","nonce":843843653},{"type":"v1","addr":"172.21.15.89:6803","nonce":843843653}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6804","nonce":843843653},{"type":"v1","addr":"172.21.15.89:6805","nonce":843843653}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6808","nonce":843843653},{"type":"v1","addr":"172.21.15.89:6809","nonce":843843653}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6806","nonce":843843653},{"type":"v1","addr":"172.21.15.89:6807","nonce":843843653}]},"public_addr":"172.21.15.89:6803/843843653","cluster_addr":"172.21.15.89:6805/843843653","heartbeat_back_addr":"172.21.15.89:6809/843843653","heartbeat_front_addr":"172.21.15.89:6807/843843653","state":["exists","up"]},{"osd":1,"uuid":"d304859e-317a-414e-a16c-f097fc4b2702","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6810","nonce":1536075263},{"type":"v1","addr":"172.21.15.89:6811","nonce":1536075263}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6812","nonce":1536075263},{"type":"v1","addr":"172.21.15.89:6813","nonce":1536075263}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6816","nonce":1536075263},{"type":"v1","addr":"172.21.15.89:6817","nonce":1536075263}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6814","nonce":1536075263},{"type":"v1","addr":"172.21.15.89:6815","nonce":1536075263}]},"public_addr":"172.21.15.89:6811/1536075263","cluster_addr":"172.21.15.89:6813/1536075263","heartbeat_back_addr":"172.21.15.89:6817/1536075263","heartbeat_front_addr":"172.21.15.89:6815/1536075263","state":["exists","up"]},{"osd":2,"uuid":"60cde686-83bb-4ca9-a0f1-6668710e16f1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6818","nonce":3236432318},{"type":"v1","addr":"172.21.15.89:6819","nonce":3236432318}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6820","nonce":3236432318},{"type":"v1","addr":"172.21.15.89:6821","nonce":3236432318}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6824","nonce":3236432318},{"type":"v1","addr":"172.21.15.89:6825","nonce":3236432318}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6822","nonce":3236432318},{"type":"v1","addr":"172.21.15.89:6823","nonce":3236432318}]},"public_addr":"172.21.15.89:6819/3236432318","cluster_addr":"172.21.15.89:6821/3236432318","heartbeat_back_addr":"172.21.15.89:6825/3236432318","heartbeat_front_addr":"172.21.15.89:6823/3236432318","state":["exists","up"]},{"osd":3,"uuid":"8af52bbd-4bc9-4b13-b525-26c92e69c29f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6826","nonce":4083663609},{"type":"v1","addr":"172.21.15.89:6827","nonce":4083663609}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6828","nonce":4083663609},{"type":"v1","addr":"172.21.15.89:6829","nonce":4083663609}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6832","nonce":4083663609},{"type":"v1","addr":"172.21.15.89:6833","nonce":4083663609}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6830","nonce":4083663609},{"type":"v1","addr":"172.21.15.89:6831","nonce":4083663609}]},"public_addr":"172.21.15.89:6827/4083663609","cluster_addr":"172.21.15.89:6829/4083663609","heartbeat_back_addr":"172.21.15.89:6833/4083663609","heartbeat_front_addr":"172.21.15.89:6831/4083663609","state":["exists","up"]},{"osd":4,"uuid":"b97874a6-e72a-4068-9c71-a50b8790be2c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6800","nonce":3588917663},{"type":"v1","addr":"172.21.15.134:6801","nonce":3588917663}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6802","nonce":3588917663},{"type":"v1","addr":"172.21.15.134:6803","nonce":3588917663}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6806","nonce":3588917663},{"type":"v1","addr":"172.21.15.134:6807","nonce":3588917663}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6804","nonce":3588917663},{"type":"v1","addr":"172.21.15.134:6805","nonce":3588917663}]},"public_addr":"172.21.15.134:6801/3588917663","cluster_addr":"172.21.15.134:6803/3588917663","heartbeat_back_addr":"172.21.15.134:6807/3588917663","heartbeat_front_addr":"172.21.15.134:6805/3588917663","state":["exists","up"]},{"osd":5,"uuid":"a834c8ee-e034-423d-ab67-1ab33bc15a97","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6808","nonce":823577783},{"type":"v1","addr":"172.21.15.134:6809","nonce":823577783}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6810","nonce":823577783},{"type":"v1","addr":"172.21.15.134:6811","nonce":823577783}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6814","nonce":823577783},{"type":"v1","addr":"172.21.15.134:6815","nonce":823577783}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6812","nonce":823577783},{"type":"v1","addr":"172.21.15.134:6813","nonce":823577783}]},"public_addr":"172.21.15.134:6809/823577783","cluster_addr":"172.21.15.134:6811/823577783","heartbeat_back_addr":"172.21.15.134:6815/823577783","heartbeat_front_addr":"172.21.15.134:6813/823577783","state":["exists","up"]},{"osd":6,"uuid":"446cb8fa-8ddb-4f7a-a8a4-c7281e950f74","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6816","nonce":4010812142},{"type":"v1","addr":"172.21.15.134:6817","nonce":4010812142}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6818","nonce":4010812142},{"type":"v1","addr":"172.21.15.134:6819","nonce":4010812142}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6822","nonce":4010812142},{"type":"v1","addr":"172.21.15.134:6823","nonce":4010812142}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6820","nonce":4010812142},{"type":"v1","addr":"172.21.15.134:6821","nonce":4010812142}]},"public_addr":"172.21.15.134:6817/4010812142","cluster_addr":"172.21.15.134:6819/4010812142","heartbeat_back_addr":"172.21.15.134:6823/4010812142","heartbeat_front_addr":"172.21.15.134:6821/4010812142","state":["exists","up"]},{"osd":7,"uuid":"87f3a30f-2935-4b92-af37-bd866696838f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6824","nonce":3883480074},{"type":"v1","addr":"172.21.15.134:6825","nonce":3883480074}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6826","nonce":3883480074},{"type":"v1","addr":"172.21.15.134:6827","nonce":3883480074}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6830","nonce":3883480074},{"type":"v1","addr":"172.21.15.134:6831","nonce":3883480074}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6828","nonce":3883480074},{"type":"v1","addr":"172.21.15.134:6829","nonce":3883480074}]},"public_addr":"172.21.15.134:6825/3883480074","cluster_addr":"172.21.15.134:6827/3883480074","heartbeat_back_addr":"172.21.15.134:6831/3883480074","heartbeat_front_addr":"172.21.15.134:6829/3883480074","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-15T06:40:36.567653+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-15T06:41:26.849575+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-15T06:42:18.731018+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-15T06:43:11.604111+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-15T06:44:03.847505+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-15T06:44:56.895305+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-15T06:45:50.039730+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-15T06:46:43.462786+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.89:6801/265242087":"2024-07-16T06:47:44.086461+0000","172.21.15.89:0/1100221668":"2024-07-16T06:47:44.086461+0000","172.21.15.89:0/1877203535":"2024-07-16T06:37:35.981306+0000","172.21.15.89:0/1750227302":"2024-07-16T06:38:06.841135+0000","172.21.15.89:6800/2189595088":"2024-07-16T06:37:35.981306+0000","172.21.15.89:6801/2189595088":"2024-07-16T06:37:35.981306+0000","172.21.15.89:0/3465936199":"2024-07-16T06:47:44.086461+0000","172.21.15.89:0/1228522973":"2024-07-16T06:37:35.981306+0000","172.21.15.89:0/889381338":"2024-07-16T06:38:06.841135+0000","172.21.15.89:0/1672418864":"2024-07-16T06:38:06.841135+0000","172.21.15.89:0/2127823469":"2024-07-16T06:37:35.981306+0000","172.21.15.89:6801/2919301358":"2024-07-16T06:38:06.841135+0000","172.21.15.89:6800/265242087":"2024-07-16T06:47:44.086461+0000","172.21.15.89:6800/2919301358":"2024-07-16T06:38:06.841135+0000","172.21.15.89:0/3680199495":"2024-07-16T06:47:44.086461+0000","172.21.15.89:0/1054475873":"2024-07-16T06:47:44.086461+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-07-15T06:48:57.738 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:57 smithi134 bash[24068]: cluster 2024-07-15T06:48:56.124318+0000 mgr.y (mgr.24272) 63 : cluster [DBG] pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:48:57.738 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:57 smithi134 bash[24068]: audit 2024-07-15T06:48:57.462458+0000 mon.c (mon.2) 20 : audit [DBG] from='client.? 172.21.15.89:0/3024492712' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-07-15T06:48:57.739 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:48:57 smithi134 bash[50669]: ceph-75ba02bc-4274-11ef-bcad-c7b262605968-prometheus-a 2024-07-15T06:48:57.739 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:48:57 smithi134 bash[50730]: Error response from daemon: No such container: ceph-75ba02bc-4274-11ef-bcad-c7b262605968-prometheus-a 2024-07-15T06:48:57.739 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:48:57 smithi134 systemd[1]: ceph-75ba02bc-4274-11ef-bcad-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-07-15T06:48:57.739 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:48:57 smithi134 systemd[1]: Stopped Ceph prometheus.a for 75ba02bc-4274-11ef-bcad-c7b262605968. 2024-07-15T06:48:57.739 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:48:57 smithi134 systemd[1]: Started Ceph prometheus.a for 75ba02bc-4274-11ef-bcad-c7b262605968. 2024-07-15T06:48:57.739 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:57 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:57.493186552Z level=info msg="Executing migration" id="create data_keys table" 2024-07-15T06:48:57.739 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:57 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:57.623043607Z level=info msg="Executing migration" id="create secrets table" 2024-07-15T06:48:57.739 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:57 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:57.698804512Z level=info msg="Executing migration" id="rename data_keys name column to id" 2024-07-15T06:48:57.817 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:57 smithi089 bash[21793]: cluster 2024-07-15T06:48:56.124318+0000 mgr.y (mgr.24272) 63 : cluster [DBG] pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:48:57.818 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:57 smithi089 bash[21793]: audit 2024-07-15T06:48:57.462458+0000 mon.c (mon.2) 20 : audit [DBG] from='client.? 172.21.15.89:0/3024492712' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-07-15T06:48:57.818 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:57 smithi089 bash[31372]: cluster 2024-07-15T06:48:56.124318+0000 mgr.y (mgr.24272) 63 : cluster [DBG] pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:48:57.818 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:57 smithi089 bash[31372]: audit 2024-07-15T06:48:57.462458+0000 mon.c (mon.2) 20 : audit [DBG] from='client.? 172.21.15.89:0/3024492712' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-07-15T06:48:58.055 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:57 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:57.766643058Z level=info msg="Executing migration" id="add name column into data_keys" 2024-07-15T06:48:58.055 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:57 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:57.817083825Z level=info msg="Executing migration" id="copy data_keys id column values into name" 2024-07-15T06:48:58.055 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:57 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:57.885122508Z level=info msg="Executing migration" id="rename data_keys name column to label" 2024-07-15T06:48:58.119 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:48:57 smithi089 bash[22045]: [15/Jul/2024:06:48:57] ENGINE Bus STOPPING 2024-07-15T06:48:58.119 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:48:58 smithi089 bash[22045]: [15/Jul/2024:06:48:58] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-07-15T06:48:58.119 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:48:58 smithi089 bash[22045]: [15/Jul/2024:06:48:58] ENGINE Bus STOPPED 2024-07-15T06:48:58.119 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:48:58 smithi089 bash[22045]: [15/Jul/2024:06:48:58] ENGINE Bus STARTING 2024-07-15T06:48:58.167 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-07-15T06:48:58.167 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph osd dump --format=json 2024-07-15T06:48:58.555 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:58 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:58.286403695Z level=info msg="Executing migration" id="rename data_keys id column back to name" 2024-07-15T06:48:58.555 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:58 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:58.370628832Z level=info msg="Executing migration" id="create kv_store table v1" 2024-07-15T06:48:58.619 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:48:58 smithi089 bash[22045]: [15/Jul/2024:06:48:58] ENGINE Serving on http://:::9283 2024-07-15T06:48:58.619 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:48:58 smithi089 bash[22045]: [15/Jul/2024:06:48:58] ENGINE Bus STARTED 2024-07-15T06:48:58.619 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:48:58 smithi089 bash[22045]: [15/Jul/2024:06:48:58] ENGINE Bus STOPPING 2024-07-15T06:48:58.819 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:58 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:58.571979036Z level=info msg="Executing migration" id="add index kv_store.org_id-namespace-key" 2024-07-15T06:48:58.819 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:58 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:58.714216879Z level=info msg="Executing migration" id="update dashboard_uid and panel_id from existing annotations" 2024-07-15T06:48:58.820 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:58 smithi134 bash[24068]: audit 2024-07-15T06:48:57.785477+0000 mon.a (mon.0) 752 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:58.820 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:58 smithi134 bash[24068]: audit 2024-07-15T06:48:57.795309+0000 mon.a (mon.0) 753 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:58.820 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:58 smithi134 bash[24068]: audit 2024-07-15T06:48:57.803276+0000 mon.a (mon.0) 754 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-07-15T06:48:58.820 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:58 smithi134 bash[24068]: audit 2024-07-15T06:48:57.804151+0000 mgr.y (mgr.24272) 64 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-07-15T06:48:58.820 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:58 smithi134 bash[24068]: audit 2024-07-15T06:48:57.806529+0000 mon.a (mon.0) 755 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi089.front.sepia.ceph.com:9093"}]: dispatch 2024-07-15T06:48:58.820 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:58 smithi134 bash[24068]: audit 2024-07-15T06:48:57.807333+0000 mgr.y (mgr.24272) 65 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi089.front.sepia.ceph.com:9093"}]: dispatch 2024-07-15T06:48:58.820 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:58 smithi134 bash[24068]: audit 2024-07-15T06:48:57.816402+0000 mon.a (mon.0) 756 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:58.820 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:58 smithi134 bash[24068]: audit 2024-07-15T06:48:57.831253+0000 mon.a (mon.0) 757 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-07-15T06:48:58.820 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:58 smithi134 bash[24068]: audit 2024-07-15T06:48:57.832002+0000 mgr.y (mgr.24272) 66 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-07-15T06:48:58.821 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:58 smithi134 bash[24068]: audit 2024-07-15T06:48:57.836541+0000 mon.a (mon.0) 758 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi134.front.sepia.ceph.com:9095"}]: dispatch 2024-07-15T06:48:58.821 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:58 smithi134 bash[24068]: audit 2024-07-15T06:48:57.837067+0000 mgr.y (mgr.24272) 67 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi134.front.sepia.ceph.com:9095"}]: dispatch 2024-07-15T06:48:58.821 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:58 smithi134 bash[24068]: audit 2024-07-15T06:48:57.842991+0000 mon.a (mon.0) 759 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:58.821 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:58 smithi134 bash[24068]: audit 2024-07-15T06:48:57.860198+0000 mon.a (mon.0) 760 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-07-15T06:48:58.821 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:58 smithi134 bash[24068]: audit 2024-07-15T06:48:57.860908+0000 mgr.y (mgr.24272) 68 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-07-15T06:48:58.821 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:58 smithi134 bash[24068]: audit 2024-07-15T06:48:57.865177+0000 mon.a (mon.0) 761 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi134.front.sepia.ceph.com:3000"}]: dispatch 2024-07-15T06:48:58.821 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:58 smithi134 bash[24068]: audit 2024-07-15T06:48:57.865667+0000 mgr.y (mgr.24272) 69 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi134.front.sepia.ceph.com:3000"}]: dispatch 2024-07-15T06:48:58.821 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:58 smithi134 bash[24068]: audit 2024-07-15T06:48:57.871731+0000 mon.a (mon.0) 762 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:58.821 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:58 smithi134 bash[24068]: audit 2024-07-15T06:48:57.880554+0000 mon.a (mon.0) 763 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:48:58.821 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:58 smithi134 bash[24068]: cluster 2024-07-15T06:48:58.124944+0000 mgr.y (mgr.24272) 70 : cluster [DBG] pgmap v44: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:48:58.821 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:58 smithi134 bash[24068]: audit 2024-07-15T06:48:58.162636+0000 mon.a (mon.0) 764 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:58.821 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:58 smithi134 bash[24068]: audit 2024-07-15T06:48:58.167356+0000 mon.a (mon.0) 765 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:59.070 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:58 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:58.823792001Z level=info msg="Executing migration" id="create permission table" 2024-07-15T06:48:59.070 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:58 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:58.936506887Z level=info msg="Executing migration" id="add unique index permission.role_id" 2024-07-15T06:48:59.070 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:59 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:59.03494746Z level=info msg="Executing migration" id="add unique index role_id_action_scope" 2024-07-15T06:48:59.070 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:59 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:59.074871092Z level=info msg="Executing migration" id="create role table" 2024-07-15T06:48:59.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:58 smithi089 bash[31372]: audit 2024-07-15T06:48:57.785477+0000 mon.a (mon.0) 752 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:59.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:58 smithi089 bash[31372]: audit 2024-07-15T06:48:57.795309+0000 mon.a (mon.0) 753 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:59.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:58 smithi089 bash[31372]: audit 2024-07-15T06:48:57.803276+0000 mon.a (mon.0) 754 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-07-15T06:48:59.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:58 smithi089 bash[31372]: audit 2024-07-15T06:48:57.804151+0000 mgr.y (mgr.24272) 64 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-07-15T06:48:59.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:58 smithi089 bash[31372]: audit 2024-07-15T06:48:57.806529+0000 mon.a (mon.0) 755 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi089.front.sepia.ceph.com:9093"}]: dispatch 2024-07-15T06:48:59.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:58 smithi089 bash[31372]: audit 2024-07-15T06:48:57.807333+0000 mgr.y (mgr.24272) 65 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi089.front.sepia.ceph.com:9093"}]: dispatch 2024-07-15T06:48:59.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:58 smithi089 bash[31372]: audit 2024-07-15T06:48:57.816402+0000 mon.a (mon.0) 756 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:59.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:58 smithi089 bash[31372]: audit 2024-07-15T06:48:57.831253+0000 mon.a (mon.0) 757 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-07-15T06:48:59.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:58 smithi089 bash[31372]: audit 2024-07-15T06:48:57.832002+0000 mgr.y (mgr.24272) 66 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-07-15T06:48:59.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:58 smithi089 bash[31372]: audit 2024-07-15T06:48:57.836541+0000 mon.a (mon.0) 758 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi134.front.sepia.ceph.com:9095"}]: dispatch 2024-07-15T06:48:59.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:58 smithi089 bash[31372]: audit 2024-07-15T06:48:57.837067+0000 mgr.y (mgr.24272) 67 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi134.front.sepia.ceph.com:9095"}]: dispatch 2024-07-15T06:48:59.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:58 smithi089 bash[31372]: audit 2024-07-15T06:48:57.842991+0000 mon.a (mon.0) 759 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:59.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:58 smithi089 bash[31372]: audit 2024-07-15T06:48:57.860198+0000 mon.a (mon.0) 760 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-07-15T06:48:59.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:58 smithi089 bash[31372]: audit 2024-07-15T06:48:57.860908+0000 mgr.y (mgr.24272) 68 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-07-15T06:48:59.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:58 smithi089 bash[31372]: audit 2024-07-15T06:48:57.865177+0000 mon.a (mon.0) 761 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi134.front.sepia.ceph.com:3000"}]: dispatch 2024-07-15T06:48:59.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:58 smithi089 bash[31372]: audit 2024-07-15T06:48:57.865667+0000 mgr.y (mgr.24272) 69 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi134.front.sepia.ceph.com:3000"}]: dispatch 2024-07-15T06:48:59.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:58 smithi089 bash[31372]: audit 2024-07-15T06:48:57.871731+0000 mon.a (mon.0) 762 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:59.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:58 smithi089 bash[31372]: audit 2024-07-15T06:48:57.880554+0000 mon.a (mon.0) 763 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:48:59.121 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:48:58 smithi089 bash[22045]: [15/Jul/2024:06:48:58] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-07-15T06:48:59.121 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:48:58 smithi089 bash[22045]: [15/Jul/2024:06:48:58] ENGINE Bus STOPPED 2024-07-15T06:48:59.121 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:48:58 smithi089 bash[22045]: [15/Jul/2024:06:48:58] ENGINE Bus STARTING 2024-07-15T06:48:59.122 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:48:58 smithi089 bash[22045]: [15/Jul/2024:06:48:58] ENGINE Serving on http://:::9283 2024-07-15T06:48:59.122 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:48:58 smithi089 bash[22045]: [15/Jul/2024:06:48:58] ENGINE Bus STARTED 2024-07-15T06:48:59.122 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:48:58 smithi089 bash[22045]: [15/Jul/2024:06:48:58] ENGINE Bus STOPPING 2024-07-15T06:48:59.122 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:48:58 smithi089 bash[22045]: [15/Jul/2024:06:48:58] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-07-15T06:48:59.122 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:48:58 smithi089 bash[22045]: [15/Jul/2024:06:48:58] ENGINE Bus STOPPED 2024-07-15T06:48:59.122 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:48:58 smithi089 bash[22045]: [15/Jul/2024:06:48:58] ENGINE Bus STARTING 2024-07-15T06:48:59.122 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:48:58 smithi089 bash[22045]: [15/Jul/2024:06:48:58] ENGINE Serving on http://:::9283 2024-07-15T06:48:59.122 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:48:58 smithi089 bash[22045]: [15/Jul/2024:06:48:58] ENGINE Bus STARTED 2024-07-15T06:48:59.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:58 smithi089 bash[31372]: cluster 2024-07-15T06:48:58.124944+0000 mgr.y (mgr.24272) 70 : cluster [DBG] pgmap v44: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:48:59.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:58 smithi089 bash[31372]: audit 2024-07-15T06:48:58.162636+0000 mon.a (mon.0) 764 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:59.123 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:58 smithi089 bash[31372]: audit 2024-07-15T06:48:58.167356+0000 mon.a (mon.0) 765 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:59.123 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:58 smithi089 bash[21793]: audit 2024-07-15T06:48:57.785477+0000 mon.a (mon.0) 752 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:59.123 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:58 smithi089 bash[21793]: audit 2024-07-15T06:48:57.795309+0000 mon.a (mon.0) 753 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:59.123 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:58 smithi089 bash[21793]: audit 2024-07-15T06:48:57.803276+0000 mon.a (mon.0) 754 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-07-15T06:48:59.123 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:58 smithi089 bash[21793]: audit 2024-07-15T06:48:57.804151+0000 mgr.y (mgr.24272) 64 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-07-15T06:48:59.123 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:58 smithi089 bash[21793]: audit 2024-07-15T06:48:57.806529+0000 mon.a (mon.0) 755 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi089.front.sepia.ceph.com:9093"}]: dispatch 2024-07-15T06:48:59.123 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:58 smithi089 bash[21793]: audit 2024-07-15T06:48:57.807333+0000 mgr.y (mgr.24272) 65 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi089.front.sepia.ceph.com:9093"}]: dispatch 2024-07-15T06:48:59.123 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:58 smithi089 bash[21793]: audit 2024-07-15T06:48:57.816402+0000 mon.a (mon.0) 756 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:59.123 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:58 smithi089 bash[21793]: audit 2024-07-15T06:48:57.831253+0000 mon.a (mon.0) 757 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-07-15T06:48:59.123 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:58 smithi089 bash[21793]: audit 2024-07-15T06:48:57.832002+0000 mgr.y (mgr.24272) 66 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-07-15T06:48:59.123 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:58 smithi089 bash[21793]: audit 2024-07-15T06:48:57.836541+0000 mon.a (mon.0) 758 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi134.front.sepia.ceph.com:9095"}]: dispatch 2024-07-15T06:48:59.124 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:58 smithi089 bash[21793]: audit 2024-07-15T06:48:57.837067+0000 mgr.y (mgr.24272) 67 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi134.front.sepia.ceph.com:9095"}]: dispatch 2024-07-15T06:48:59.124 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:58 smithi089 bash[21793]: audit 2024-07-15T06:48:57.842991+0000 mon.a (mon.0) 759 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:59.124 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:58 smithi089 bash[21793]: audit 2024-07-15T06:48:57.860198+0000 mon.a (mon.0) 760 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-07-15T06:48:59.124 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:58 smithi089 bash[21793]: audit 2024-07-15T06:48:57.860908+0000 mgr.y (mgr.24272) 68 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-07-15T06:48:59.124 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:58 smithi089 bash[21793]: audit 2024-07-15T06:48:57.865177+0000 mon.a (mon.0) 761 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi134.front.sepia.ceph.com:3000"}]: dispatch 2024-07-15T06:48:59.124 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:58 smithi089 bash[21793]: audit 2024-07-15T06:48:57.865667+0000 mgr.y (mgr.24272) 69 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi134.front.sepia.ceph.com:3000"}]: dispatch 2024-07-15T06:48:59.124 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:58 smithi089 bash[21793]: audit 2024-07-15T06:48:57.871731+0000 mon.a (mon.0) 762 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:59.124 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:58 smithi089 bash[21793]: audit 2024-07-15T06:48:57.880554+0000 mon.a (mon.0) 763 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:48:59.124 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:58 smithi089 bash[21793]: cluster 2024-07-15T06:48:58.124944+0000 mgr.y (mgr.24272) 70 : cluster [DBG] pgmap v44: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:48:59.124 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:58 smithi089 bash[21793]: audit 2024-07-15T06:48:58.162636+0000 mon.a (mon.0) 764 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:59.124 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:58 smithi089 bash[21793]: audit 2024-07-15T06:48:58.167356+0000 mon.a (mon.0) 765 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:48:59.336 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:59 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:59.151737377Z level=info msg="Executing migration" id="add column display_name" 2024-07-15T06:48:59.336 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:59 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:59.217327016Z level=info msg="Executing migration" id="add column group_name" 2024-07-15T06:48:59.336 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:59 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:59.259440945Z level=info msg="Executing migration" id="add index role.org_id" 2024-07-15T06:48:59.337 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:59 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:59.301615695Z level=info msg="Executing migration" id="add unique index role_org_id_name" 2024-07-15T06:48:59.592 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:59 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:59.343564281Z level=info msg="Executing migration" id="add index role_org_id_uid" 2024-07-15T06:48:59.593 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:59 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:59.411327553Z level=info msg="Executing migration" id="create team role table" 2024-07-15T06:48:59.593 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:59 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:59.436596856Z level=info msg="Executing migration" id="add index team_role.org_id" 2024-07-15T06:48:59.593 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:59 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:59.453768291Z level=info msg="Executing migration" id="add unique index team_role_org_id_team_id_role_id" 2024-07-15T06:48:59.593 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:59 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:59.470711021Z level=info msg="Executing migration" id="add index team_role.team_id" 2024-07-15T06:48:59.593 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:59 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:59.48762209Z level=info msg="Executing migration" id="create user role table" 2024-07-15T06:48:59.593 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:59 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:59.504556765Z level=info msg="Executing migration" id="add index user_role.org_id" 2024-07-15T06:48:59.593 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:59 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:59.521417743Z level=info msg="Executing migration" id="add unique index user_role_org_id_user_id_role_id" 2024-07-15T06:48:59.593 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:59 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:59.546754895Z level=info msg="Executing migration" id="add index user_role.user_id" 2024-07-15T06:48:59.593 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:59 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:59.572028018Z level=info msg="Executing migration" id="create builtin role table" 2024-07-15T06:48:59.594 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:48:59 smithi134 bash[50757]: ts=2024-07-15T06:48:59.341Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-07-15T06:48:59.594 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:48:59 smithi134 bash[50757]: ts=2024-07-15T06:48:59.341Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-07-15T06:48:59.594 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:48:59 smithi134 bash[50757]: ts=2024-07-15T06:48:59.341Z caller=main.go:570 level=info host_details="(Linux 5.15.0-113-generic #123-Ubuntu SMP Mon Jun 10 08:16:17 UTC 2024 x86_64 smithi134 (none))" 2024-07-15T06:48:59.594 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:48:59 smithi134 bash[50757]: ts=2024-07-15T06:48:59.341Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-07-15T06:48:59.594 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:48:59 smithi134 bash[50757]: ts=2024-07-15T06:48:59.341Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-07-15T06:48:59.594 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:48:59 smithi134 bash[50757]: ts=2024-07-15T06:48:59.343Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-07-15T06:48:59.594 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:48:59 smithi134 bash[50757]: ts=2024-07-15T06:48:59.344Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-07-15T06:48:59.594 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:48:59 smithi134 bash[50757]: ts=2024-07-15T06:48:59.344Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-07-15T06:48:59.594 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:48:59 smithi134 bash[50757]: ts=2024-07-15T06:48:59.344Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-07-15T06:48:59.595 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:48:59 smithi134 bash[50757]: ts=2024-07-15T06:48:59.348Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-07-15T06:48:59.595 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:48:59 smithi134 bash[50757]: ts=2024-07-15T06:48:59.348Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.945µs 2024-07-15T06:48:59.595 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:48:59 smithi134 bash[50757]: ts=2024-07-15T06:48:59.348Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-07-15T06:48:59.595 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:48:59 smithi134 bash[50757]: ts=2024-07-15T06:48:59.349Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=1 2024-07-15T06:48:59.595 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:48:59 smithi134 bash[50757]: ts=2024-07-15T06:48:59.349Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=1 2024-07-15T06:48:59.595 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:48:59 smithi134 bash[50757]: ts=2024-07-15T06:48:59.349Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=30.435µs wal_replay_duration=646.388µs wbl_replay_duration=106ns total_replay_duration=698.201µs 2024-07-15T06:48:59.595 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:48:59 smithi134 bash[50757]: ts=2024-07-15T06:48:59.350Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-07-15T06:48:59.595 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:48:59 smithi134 bash[50757]: ts=2024-07-15T06:48:59.350Z caller=main.go:1029 level=info msg="TSDB started" 2024-07-15T06:48:59.595 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:48:59 smithi134 bash[50757]: ts=2024-07-15T06:48:59.350Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-07-15T06:48:59.595 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:48:59 smithi134 bash[50757]: ts=2024-07-15T06:48:59.368Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=18.033739ms db_storage=882ns remote_storage=1.625µs web_handler=428ns query_engine=683ns scrape=294.582µs scrape_sd=30.328µs notify=35.261µs notify_sd=9.43µs rules=17.387625ms tracing=5.882µs 2024-07-15T06:48:59.595 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:48:59 smithi134 bash[50757]: ts=2024-07-15T06:48:59.368Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-07-15T06:48:59.595 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:48:59 smithi134 bash[50757]: ts=2024-07-15T06:48:59.368Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-07-15T06:48:59.862 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:59 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:59.597268315Z level=info msg="Executing migration" id="add index builtin_role.role_id" 2024-07-15T06:48:59.862 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:59 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:59.622553241Z level=info msg="Executing migration" id="add index builtin_role.name" 2024-07-15T06:48:59.862 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:59 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:59.647795848Z level=info msg="Executing migration" id="Add column org_id to builtin_role table" 2024-07-15T06:48:59.862 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:59 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:59.673054565Z level=info msg="Executing migration" id="add index builtin_role.org_id" 2024-07-15T06:48:59.862 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:59 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:59.698271608Z level=info msg="Executing migration" id="add unique index builtin_role_org_id_role_id_role" 2024-07-15T06:48:59.862 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:59 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:59.740337947Z level=info msg="Executing migration" id="Remove unique index role_org_id_uid" 2024-07-15T06:48:59.863 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:59 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:59.765616046Z level=info msg="Executing migration" id="add unique index role.uid" 2024-07-15T06:48:59.863 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:59 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:59.799261209Z level=info msg="Executing migration" id="create seed assignment table" 2024-07-15T06:48:59.863 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:59 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:59.816203628Z level=info msg="Executing migration" id="add unique index builtin_role_role_name" 2024-07-15T06:48:59.863 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:59 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:59.833160947Z level=info msg="Executing migration" id="add column hidden to role table" 2024-07-15T06:48:59.863 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:59 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:59.866792373Z level=info msg="Executing migration" id="create query_history table v1" 2024-07-15T06:49:00.114 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:48:59 smithi134 bash[24068]: audit 2024-07-15T06:48:59.189911+0000 mon.a (mon.0) 766 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:49:00.114 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:59 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:59.908848725Z level=info msg="Executing migration" id="add index query_history.org_id-created_by-datasource_uid" 2024-07-15T06:49:00.114 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:59 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:59.925773358Z level=info msg="Executing migration" id="alter table query_history alter column created_by type to bigint" 2024-07-15T06:49:00.115 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:48:59 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:48:59.976026757Z level=info msg="Executing migration" id="teams permissions migration" 2024-07-15T06:49:00.115 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:49:00 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:49:00.001294766Z level=info msg="Executing migration" id="dashboard permissions" 2024-07-15T06:49:00.115 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:49:00 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:49:00.02655839Z level=info msg="Executing migration" id="dashboard permissions uid scopes" 2024-07-15T06:49:00.115 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:49:00 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:49:00.051800708Z level=info msg="Executing migration" id="drop managed folder create actions" 2024-07-15T06:49:00.115 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:49:00 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:49:00.077120107Z level=info msg="Executing migration" id="alerting notification permissions" 2024-07-15T06:49:00.115 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:49:00 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:49:00.119122481Z level=info msg="Executing migration" id="create query_history_star table v1" 2024-07-15T06:49:00.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:48:59 smithi089 bash[21793]: audit 2024-07-15T06:48:59.189911+0000 mon.a (mon.0) 766 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:49:00.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:48:59 smithi089 bash[31372]: audit 2024-07-15T06:48:59.189911+0000 mon.a (mon.0) 766 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:49:00.401 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:49:00 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:49:00.161261894Z level=info msg="Executing migration" id="add index query_history.user_id-query_uid" 2024-07-15T06:49:00.401 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:49:00 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:49:00.186513875Z level=info msg="Executing migration" id="add column org_id in query_history_star" 2024-07-15T06:49:00.401 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:49:00 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:49:00.21170075Z level=info msg="Executing migration" id="alter table query_history_star_mig column user_id type to bigint" 2024-07-15T06:49:00.401 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:49:00 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:49:00.237267366Z level=info msg="Executing migration" id="create correlation table v1" 2024-07-15T06:49:00.401 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:49:00 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:49:00.270931572Z level=info msg="Executing migration" id="add index correlations.uid" 2024-07-15T06:49:00.401 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:49:00 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:49:00.304692794Z level=info msg="Executing migration" id="add index correlations.source_uid" 2024-07-15T06:49:00.401 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:49:00 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:49:00.321589011Z level=info msg="Executing migration" id="add correlation config column" 2024-07-15T06:49:00.401 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:49:00 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:49:00.338480099Z level=info msg="Executing migration" id="create entity_events table" 2024-07-15T06:49:00.401 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:49:00 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:49:00.363853421Z level=info msg="Executing migration" id="create dashboard public config v1" 2024-07-15T06:49:00.402 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:49:00 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:49:00.405654595Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v1" 2024-07-15T06:49:00.402 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:49:00 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:49:00.40595165Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index UQE_dashboard_public_config_uid - v1" 2024-07-15T06:49:00.662 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:49:00 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:49:00.43106634Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-07-15T06:49:00.662 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:49:00 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:49:00.431411354Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-07-15T06:49:00.662 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:49:00 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:49:00.456310092Z level=info msg="Executing migration" id="Drop old dashboard public config table" 2024-07-15T06:49:00.662 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:49:00 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:49:00.48167668Z level=info msg="Executing migration" id="recreate dashboard public config v1" 2024-07-15T06:49:00.662 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:49:00 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:49:00.515197367Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v1" 2024-07-15T06:49:00.663 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:49:00 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:49:00.540459311Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-07-15T06:49:00.663 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:49:00 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:49:00.565692679Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v2" 2024-07-15T06:49:00.663 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:49:00 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:49:00.590944234Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-07-15T06:49:00.663 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:49:00 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:49:00.616228694Z level=info msg="Executing migration" id="Drop public config table" 2024-07-15T06:49:00.663 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:49:00 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:49:00.64154356Z level=info msg="Executing migration" id="Recreate dashboard public config v2" 2024-07-15T06:49:00.663 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:49:00 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:49:00.666753803Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v2" 2024-07-15T06:49:00.939 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:49:00 smithi134 bash[24068]: cluster 2024-07-15T06:49:00.125797+0000 mgr.y (mgr.24272) 71 : cluster [DBG] pgmap v45: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:49:00.940 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:49:00 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:49:00.708676905Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-07-15T06:49:00.940 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:49:00 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:49:00.725693573Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_access_token - v2" 2024-07-15T06:49:00.940 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:49:00 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:49:00.75936612Z level=info msg="Executing migration" id="Rename table dashboard_public_config to dashboard_public - v2" 2024-07-15T06:49:00.940 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:49:00 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:49:00.784637704Z level=info msg="Executing migration" id="add annotations_enabled column" 2024-07-15T06:49:00.940 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:49:00 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:49:00.818277828Z level=info msg="Executing migration" id="add time_selection_enabled column" 2024-07-15T06:49:00.940 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:49:00 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:49:00.860318606Z level=info msg="Executing migration" id="delete orphaned public dashboards" 2024-07-15T06:49:00.940 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:49:00 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:49:00.902473616Z level=info msg="Executing migration" id="add share column" 2024-07-15T06:49:00.940 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:49:00 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:49:00.944449591Z level=info msg="Executing migration" id="create default alerting folders" 2024-07-15T06:49:01.193 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:49:00 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:49:00.969733055Z level=info msg="Executing migration" id="create file table" 2024-07-15T06:49:01.193 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:49:00 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:49:00.995066768Z level=info msg="Executing migration" id="file table idx: path natural pk" 2024-07-15T06:49:01.193 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:49:01 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:49:01.020575972Z level=info msg="Executing migration" id="file table idx: parent_folder_path_hash fast folder retrieval" 2024-07-15T06:49:01.193 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:49:01 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:49:01.045908402Z level=info msg="Executing migration" id="create file_meta table" 2024-07-15T06:49:01.193 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:49:01 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:49:01.087819966Z level=info msg="Executing migration" id="file table idx: path key" 2024-07-15T06:49:01.193 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:49:01 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:49:01.113126347Z level=info msg="Executing migration" id="set path collation in file table" 2024-07-15T06:49:01.193 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:49:01 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:49:01.138335486Z level=info msg="Executing migration" id="managed permissions migration" 2024-07-15T06:49:01.193 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:49:01 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:49:01.163552679Z level=info msg="Executing migration" id="managed folder permissions alert actions migration" 2024-07-15T06:49:01.193 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:49:01 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:49:01.180468657Z level=info msg="Executing migration" id="RBAC action name migrator" 2024-07-15T06:49:01.193 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:49:01 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:49:01.19742619Z level=info msg="Executing migration" id="Add UID column to playlist" 2024-07-15T06:49:01.369 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:49:00 smithi089 bash[21793]: cluster 2024-07-15T06:49:00.125797+0000 mgr.y (mgr.24272) 71 : cluster [DBG] pgmap v45: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:49:01.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:49:00 smithi089 bash[31372]: cluster 2024-07-15T06:49:00.125797+0000 mgr.y (mgr.24272) 71 : cluster [DBG] pgmap v45: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:49:01.462 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:49:01 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:49:01.214302012Z level=info msg="Executing migration" id="Update uid column values in playlist" 2024-07-15T06:49:01.462 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:49:01 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:49:01.239633032Z level=info msg="Executing migration" id="Add index for uid in playlist" 2024-07-15T06:49:01.462 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:49:01 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:49:01.281644068Z level=info msg="Executing migration" id="update group index for alert rules" 2024-07-15T06:49:01.462 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:49:01 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:49:01.323563182Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated migration" 2024-07-15T06:49:01.462 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:49:01 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:49:01.348845497Z level=info msg="Executing migration" id="admin only folder/dashboard permission" 2024-07-15T06:49:01.462 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:49:01 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:49:01.390905841Z level=info msg="Executing migration" id="add action column to seed_assignment" 2024-07-15T06:49:01.462 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:49:01 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:49:01.416146858Z level=info msg="Executing migration" id="add scope column to seed_assignment" 2024-07-15T06:49:01.462 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:49:01 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:49:01.441483021Z level=info msg="Executing migration" id="remove unique index builtin_role_role_name before nullable update" 2024-07-15T06:49:01.463 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:49:01 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:49:01.466666715Z level=info msg="Executing migration" id="update seed_assignment role_name column to nullable" 2024-07-15T06:49:01.714 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:49:01 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:49:01.5336126Z level=info msg="Executing migration" id="add unique index builtin_role_name back" 2024-07-15T06:49:01.714 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:49:01 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:49:01.558877126Z level=info msg="Executing migration" id="add unique index builtin_role_action_scope" 2024-07-15T06:49:01.714 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:49:01 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:49:01.584205027Z level=info msg="Executing migration" id="add primary key to seed_assigment" 2024-07-15T06:49:01.714 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:49:01 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:49:01.626082275Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated fixed migration" 2024-07-15T06:49:01.714 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:49:01 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:49:01.651307835Z level=info msg="Executing migration" id="migrate external alertmanagers to datsourcse" 2024-07-15T06:49:01.714 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:49:01 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:49:01.684992896Z level=info msg="Executing migration" id="create folder table" 2024-07-15T06:49:01.714 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:49:01 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:49:01.718602625Z level=info msg="Executing migration" id="Add index for parent_uid" 2024-07-15T06:49:01.985 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:49:01 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:49:01.735587441Z level=info msg="Executing migration" id="Add unique index for folder.uid and folder.org_id" 2024-07-15T06:49:01.985 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:49:01 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:49:01.785896689Z level=info msg="Executing migration" id="Update folder title length" 2024-07-15T06:49:01.985 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:49:01 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:49:01.811171654Z level=info msg="Executing migration" id="Add unique index for folder.title and folder.parent_uid" 2024-07-15T06:49:01.985 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:49:01 smithi134 bash[50088]: logger=migrator t=2024-07-15T06:49:01.836430059Z level=info msg="migrations completed" performed=485 skipped=0 duration=21.426396056s 2024-07-15T06:49:01.985 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:49:01 smithi134 bash[50088]: logger=sqlstore t=2024-07-15T06:49:01.83781862Z level=info msg="Created default organization" 2024-07-15T06:49:01.985 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:49:01 smithi134 bash[50088]: logger=plugin.loader t=2024-07-15T06:49:01.960877268Z level=info msg="Plugin registered" pluginID=input 2024-07-15T06:49:01.985 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:49:01 smithi134 bash[50088]: logger=plugin.loader t=2024-07-15T06:49:01.98992918Z level=info msg="Plugin registered" pluginID=grafana-piechart-panel 2024-07-15T06:49:02.119 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 06:49:01 smithi089 bash[58097]: ts=2024-07-15T06:49:01.793Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.001879097s 2024-07-15T06:49:02.305 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:49:01 smithi134 bash[50088]: logger=plugin.loader t=2024-07-15T06:49:01.989951794Z level=info msg="Plugin registered" pluginID=vonage-status-panel 2024-07-15T06:49:02.305 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:49:01 smithi134 bash[50088]: logger=secrets t=2024-07-15T06:49:01.989989925Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2024-07-15T06:49:02.305 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:49:01 smithi134 bash[50088]: logger=query_data t=2024-07-15T06:49:01.991978379Z level=info msg="Query Service initialization" 2024-07-15T06:49:02.305 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:49:01 smithi134 bash[50088]: logger=live.push_http t=2024-07-15T06:49:01.995087329Z level=info msg="Live Push Gateway initialization" 2024-07-15T06:49:02.693 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T06:49:03.459 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:49:03 smithi134 bash[24068]: cluster 2024-07-15T06:49:02.126345+0000 mgr.y (mgr.24272) 72 : cluster [DBG] pgmap v46: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:49:03.459 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:49:03 smithi134 bash[50088]: logger=infra.usagestats.collector t=2024-07-15T06:49:03.26052797Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2024-07-15T06:49:03.459 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:49:03 smithi134 bash[50088]: logger=provisioning.datasources t=2024-07-15T06:49:03.261316755Z level=warn msg="[Deprecated] the datasource provisioning config is outdated. please upgrade" filename=/etc/grafana/provisioning/datasources/ceph-dashboard.yml 2024-07-15T06:49:03.459 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:49:03 smithi134 bash[50088]: logger=provisioning.datasources t=2024-07-15T06:49:03.295118247Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2024-07-15T06:49:03.459 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:49:03 smithi134 bash[50088]: logger=provisioning.datasources t=2024-07-15T06:49:03.398344303Z level=info msg="inserting datasource from configuration " name=Loki uid=P8E80F9AEF21F6940 2024-07-15T06:49:03.460 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:49:03 smithi134 bash[50088]: logger=provisioning.alerting t=2024-07-15T06:49:03.46381029Z level=info msg="starting to provision alerting" 2024-07-15T06:49:03.460 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:49:03 smithi134 bash[50088]: logger=provisioning.alerting t=2024-07-15T06:49:03.463863814Z level=info msg="finished to provision alerting" 2024-07-15T06:49:03.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:49:03 smithi089 bash[21793]: cluster 2024-07-15T06:49:02.126345+0000 mgr.y (mgr.24272) 72 : cluster [DBG] pgmap v46: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:49:03.619 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:49:03 smithi089 bash[31372]: cluster 2024-07-15T06:49:02.126345+0000 mgr.y (mgr.24272) 72 : cluster [DBG] pgmap v46: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:49:03.805 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:49:03 smithi134 bash[50088]: logger=grafanaStorageLogger t=2024-07-15T06:49:03.464151119Z level=info msg="storage starting" 2024-07-15T06:49:03.805 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:49:03 smithi134 bash[50088]: logger=ngalert.state.manager t=2024-07-15T06:49:03.465061024Z level=info msg="Warming state cache for startup" 2024-07-15T06:49:03.805 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:49:03 smithi134 bash[50088]: logger=http.server t=2024-07-15T06:49:03.471819381Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2024-07-15T06:49:03.805 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:49:03 smithi134 bash[50088]: logger=sqlstore.transactions t=2024-07-15T06:49:03.476024972Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-07-15T06:49:03.805 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:49:03 smithi134 bash[50088]: logger=ngalert.state.manager t=2024-07-15T06:49:03.592973534Z level=info msg="State cache has been initialized" states=0 duration=127.909194ms 2024-07-15T06:49:03.806 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:49:03 smithi134 bash[50088]: logger=ngalert.multiorg.alertmanager t=2024-07-15T06:49:03.593067269Z level=info msg="starting MultiOrg Alertmanager" 2024-07-15T06:49:03.806 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:49:03 smithi134 bash[50088]: logger=ticker t=2024-07-15T06:49:03.593117398Z level=info msg=starting first_tick=2024-07-15T06:49:10Z 2024-07-15T06:49:04.205 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:49:03 smithi134 bash[50088]: logger=sqlstore.transactions t=2024-07-15T06:49:03.941765671Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-07-15T06:49:04.369 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:49:04 smithi089 bash[21793]: audit 2024-07-15T06:49:03.210040+0000 mon.a (mon.0) 767 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:49:04.369 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:49:04 smithi089 bash[21793]: audit 2024-07-15T06:49:03.214894+0000 mon.a (mon.0) 768 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:49:04.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:49:04 smithi089 bash[31372]: audit 2024-07-15T06:49:03.210040+0000 mon.a (mon.0) 767 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:49:04.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:49:04 smithi089 bash[31372]: audit 2024-07-15T06:49:03.214894+0000 mon.a (mon.0) 768 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:49:04.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:49:04 smithi134 bash[24068]: audit 2024-07-15T06:49:03.210040+0000 mon.a (mon.0) 767 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:49:04.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:49:04 smithi134 bash[24068]: audit 2024-07-15T06:49:03.214894+0000 mon.a (mon.0) 768 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:49:05.997 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:49:05 smithi089 bash[21793]: cluster 2024-07-15T06:49:04.126750+0000 mgr.y (mgr.24272) 73 : cluster [DBG] pgmap v47: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:49:05.998 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:49:05 smithi089 bash[21793]: audit 2024-07-15T06:49:04.738854+0000 mon.a (mon.0) 769 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:49:05.998 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:49:05 smithi089 bash[21793]: audit 2024-07-15T06:49:04.750371+0000 mon.a (mon.0) 770 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:49:05.998 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:49:05 smithi089 bash[21793]: audit 2024-07-15T06:49:05.142035+0000 mon.a (mon.0) 771 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:49:05.998 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:49:05 smithi089 bash[21793]: audit 2024-07-15T06:49:05.143219+0000 mon.a (mon.0) 772 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:49:05.998 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:49:05 smithi089 bash[21793]: audit 2024-07-15T06:49:05.153661+0000 mon.a (mon.0) 773 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:49:05.998 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:49:05 smithi089 bash[21793]: audit 2024-07-15T06:49:05.197992+0000 mon.a (mon.0) 774 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:49:05.998 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:49:05 smithi089 bash[21793]: audit 2024-07-15T06:49:05.199876+0000 mon.a (mon.0) 775 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:49:05.998 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:49:05 smithi089 bash[21793]: audit 2024-07-15T06:49:05.200911+0000 mon.a (mon.0) 776 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:49:05.998 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:49:05 smithi089 bash[21793]: audit 2024-07-15T06:49:05.207610+0000 mon.a (mon.0) 777 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:49:05.999 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:49:05 smithi089 bash[31372]: cluster 2024-07-15T06:49:04.126750+0000 mgr.y (mgr.24272) 73 : cluster [DBG] pgmap v47: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:49:05.999 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:49:05 smithi089 bash[31372]: audit 2024-07-15T06:49:04.738854+0000 mon.a (mon.0) 769 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:49:05.999 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:49:05 smithi089 bash[31372]: audit 2024-07-15T06:49:04.750371+0000 mon.a (mon.0) 770 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:49:05.999 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:49:05 smithi089 bash[31372]: audit 2024-07-15T06:49:05.142035+0000 mon.a (mon.0) 771 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:49:05.999 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:49:05 smithi089 bash[31372]: audit 2024-07-15T06:49:05.143219+0000 mon.a (mon.0) 772 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:49:05.999 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:49:05 smithi089 bash[31372]: audit 2024-07-15T06:49:05.153661+0000 mon.a (mon.0) 773 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:49:05.999 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:49:05 smithi089 bash[31372]: audit 2024-07-15T06:49:05.197992+0000 mon.a (mon.0) 774 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:49:05.999 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:49:05 smithi089 bash[31372]: audit 2024-07-15T06:49:05.199876+0000 mon.a (mon.0) 775 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:49:05.999 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:49:05 smithi089 bash[31372]: audit 2024-07-15T06:49:05.200911+0000 mon.a (mon.0) 776 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:49:05.999 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:49:05 smithi089 bash[31372]: audit 2024-07-15T06:49:05.207610+0000 mon.a (mon.0) 777 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:49:06.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:49:05 smithi134 bash[24068]: cluster 2024-07-15T06:49:04.126750+0000 mgr.y (mgr.24272) 73 : cluster [DBG] pgmap v47: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:49:06.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:49:05 smithi134 bash[24068]: audit 2024-07-15T06:49:04.738854+0000 mon.a (mon.0) 769 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:49:06.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:49:05 smithi134 bash[24068]: audit 2024-07-15T06:49:04.750371+0000 mon.a (mon.0) 770 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:49:06.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:49:05 smithi134 bash[24068]: audit 2024-07-15T06:49:05.142035+0000 mon.a (mon.0) 771 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:49:06.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:49:05 smithi134 bash[24068]: audit 2024-07-15T06:49:05.143219+0000 mon.a (mon.0) 772 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:49:06.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:49:05 smithi134 bash[24068]: audit 2024-07-15T06:49:05.153661+0000 mon.a (mon.0) 773 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:49:06.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:49:05 smithi134 bash[24068]: audit 2024-07-15T06:49:05.197992+0000 mon.a (mon.0) 774 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:49:06.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:49:05 smithi134 bash[24068]: audit 2024-07-15T06:49:05.199876+0000 mon.a (mon.0) 775 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:49:06.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:49:05 smithi134 bash[24068]: audit 2024-07-15T06:49:05.200911+0000 mon.a (mon.0) 776 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:49:06.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:49:05 smithi134 bash[24068]: audit 2024-07-15T06:49:05.207610+0000 mon.a (mon.0) 777 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:49:06.333 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-07-15T06:49:06.334 INFO:teuthology.orchestra.run.smithi089.stdout:{"epoch":63,"fsid":"75ba02bc-4274-11ef-bcad-c7b262605968","created":"2024-07-15T06:37:09.634877+0000","modified":"2024-07-15T06:48:36.696104+0000","last_up_change":"2024-07-15T06:46:45.306680+0000","last_in_change":"2024-07-15T06:46:19.019927+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-07-15T06:42:21.444347+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2024-07-15T06:48:27.276344+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"79357114-251f-489d-bef5-f6ec37002212","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6802","nonce":843843653},{"type":"v1","addr":"172.21.15.89:6803","nonce":843843653}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6804","nonce":843843653},{"type":"v1","addr":"172.21.15.89:6805","nonce":843843653}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6808","nonce":843843653},{"type":"v1","addr":"172.21.15.89:6809","nonce":843843653}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6806","nonce":843843653},{"type":"v1","addr":"172.21.15.89:6807","nonce":843843653}]},"public_addr":"172.21.15.89:6803/843843653","cluster_addr":"172.21.15.89:6805/843843653","heartbeat_back_addr":"172.21.15.89:6809/843843653","heartbeat_front_addr":"172.21.15.89:6807/843843653","state":["exists","up"]},{"osd":1,"uuid":"d304859e-317a-414e-a16c-f097fc4b2702","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6810","nonce":1536075263},{"type":"v1","addr":"172.21.15.89:6811","nonce":1536075263}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6812","nonce":1536075263},{"type":"v1","addr":"172.21.15.89:6813","nonce":1536075263}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6816","nonce":1536075263},{"type":"v1","addr":"172.21.15.89:6817","nonce":1536075263}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6814","nonce":1536075263},{"type":"v1","addr":"172.21.15.89:6815","nonce":1536075263}]},"public_addr":"172.21.15.89:6811/1536075263","cluster_addr":"172.21.15.89:6813/1536075263","heartbeat_back_addr":"172.21.15.89:6817/1536075263","heartbeat_front_addr":"172.21.15.89:6815/1536075263","state":["exists","up"]},{"osd":2,"uuid":"60cde686-83bb-4ca9-a0f1-6668710e16f1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6818","nonce":3236432318},{"type":"v1","addr":"172.21.15.89:6819","nonce":3236432318}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6820","nonce":3236432318},{"type":"v1","addr":"172.21.15.89:6821","nonce":3236432318}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6824","nonce":3236432318},{"type":"v1","addr":"172.21.15.89:6825","nonce":3236432318}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6822","nonce":3236432318},{"type":"v1","addr":"172.21.15.89:6823","nonce":3236432318}]},"public_addr":"172.21.15.89:6819/3236432318","cluster_addr":"172.21.15.89:6821/3236432318","heartbeat_back_addr":"172.21.15.89:6825/3236432318","heartbeat_front_addr":"172.21.15.89:6823/3236432318","state":["exists","up"]},{"osd":3,"uuid":"8af52bbd-4bc9-4b13-b525-26c92e69c29f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6826","nonce":4083663609},{"type":"v1","addr":"172.21.15.89:6827","nonce":4083663609}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6828","nonce":4083663609},{"type":"v1","addr":"172.21.15.89:6829","nonce":4083663609}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6832","nonce":4083663609},{"type":"v1","addr":"172.21.15.89:6833","nonce":4083663609}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6830","nonce":4083663609},{"type":"v1","addr":"172.21.15.89:6831","nonce":4083663609}]},"public_addr":"172.21.15.89:6827/4083663609","cluster_addr":"172.21.15.89:6829/4083663609","heartbeat_back_addr":"172.21.15.89:6833/4083663609","heartbeat_front_addr":"172.21.15.89:6831/4083663609","state":["exists","up"]},{"osd":4,"uuid":"b97874a6-e72a-4068-9c71-a50b8790be2c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6800","nonce":3588917663},{"type":"v1","addr":"172.21.15.134:6801","nonce":3588917663}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6802","nonce":3588917663},{"type":"v1","addr":"172.21.15.134:6803","nonce":3588917663}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6806","nonce":3588917663},{"type":"v1","addr":"172.21.15.134:6807","nonce":3588917663}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6804","nonce":3588917663},{"type":"v1","addr":"172.21.15.134:6805","nonce":3588917663}]},"public_addr":"172.21.15.134:6801/3588917663","cluster_addr":"172.21.15.134:6803/3588917663","heartbeat_back_addr":"172.21.15.134:6807/3588917663","heartbeat_front_addr":"172.21.15.134:6805/3588917663","state":["exists","up"]},{"osd":5,"uuid":"a834c8ee-e034-423d-ab67-1ab33bc15a97","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6808","nonce":823577783},{"type":"v1","addr":"172.21.15.134:6809","nonce":823577783}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6810","nonce":823577783},{"type":"v1","addr":"172.21.15.134:6811","nonce":823577783}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6814","nonce":823577783},{"type":"v1","addr":"172.21.15.134:6815","nonce":823577783}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6812","nonce":823577783},{"type":"v1","addr":"172.21.15.134:6813","nonce":823577783}]},"public_addr":"172.21.15.134:6809/823577783","cluster_addr":"172.21.15.134:6811/823577783","heartbeat_back_addr":"172.21.15.134:6815/823577783","heartbeat_front_addr":"172.21.15.134:6813/823577783","state":["exists","up"]},{"osd":6,"uuid":"446cb8fa-8ddb-4f7a-a8a4-c7281e950f74","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6816","nonce":4010812142},{"type":"v1","addr":"172.21.15.134:6817","nonce":4010812142}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6818","nonce":4010812142},{"type":"v1","addr":"172.21.15.134:6819","nonce":4010812142}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6822","nonce":4010812142},{"type":"v1","addr":"172.21.15.134:6823","nonce":4010812142}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6820","nonce":4010812142},{"type":"v1","addr":"172.21.15.134:6821","nonce":4010812142}]},"public_addr":"172.21.15.134:6817/4010812142","cluster_addr":"172.21.15.134:6819/4010812142","heartbeat_back_addr":"172.21.15.134:6823/4010812142","heartbeat_front_addr":"172.21.15.134:6821/4010812142","state":["exists","up"]},{"osd":7,"uuid":"87f3a30f-2935-4b92-af37-bd866696838f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6824","nonce":3883480074},{"type":"v1","addr":"172.21.15.134:6825","nonce":3883480074}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6826","nonce":3883480074},{"type":"v1","addr":"172.21.15.134:6827","nonce":3883480074}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6830","nonce":3883480074},{"type":"v1","addr":"172.21.15.134:6831","nonce":3883480074}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6828","nonce":3883480074},{"type":"v1","addr":"172.21.15.134:6829","nonce":3883480074}]},"public_addr":"172.21.15.134:6825/3883480074","cluster_addr":"172.21.15.134:6827/3883480074","heartbeat_back_addr":"172.21.15.134:6831/3883480074","heartbeat_front_addr":"172.21.15.134:6829/3883480074","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-15T06:40:36.567653+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-15T06:41:26.849575+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-15T06:42:18.731018+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-15T06:43:11.604111+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-15T06:44:03.847505+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-15T06:44:56.895305+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-15T06:45:50.039730+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-15T06:46:43.462786+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.89:6801/265242087":"2024-07-16T06:47:44.086461+0000","172.21.15.89:0/1100221668":"2024-07-16T06:47:44.086461+0000","172.21.15.89:0/1877203535":"2024-07-16T06:37:35.981306+0000","172.21.15.89:0/1750227302":"2024-07-16T06:38:06.841135+0000","172.21.15.89:6800/2189595088":"2024-07-16T06:37:35.981306+0000","172.21.15.89:6801/2189595088":"2024-07-16T06:37:35.981306+0000","172.21.15.89:0/3465936199":"2024-07-16T06:47:44.086461+0000","172.21.15.89:0/1228522973":"2024-07-16T06:37:35.981306+0000","172.21.15.89:0/889381338":"2024-07-16T06:38:06.841135+0000","172.21.15.89:0/1672418864":"2024-07-16T06:38:06.841135+0000","172.21.15.89:0/2127823469":"2024-07-16T06:37:35.981306+0000","172.21.15.89:6801/2919301358":"2024-07-16T06:38:06.841135+0000","172.21.15.89:6800/265242087":"2024-07-16T06:47:44.086461+0000","172.21.15.89:6800/2919301358":"2024-07-16T06:38:06.841135+0000","172.21.15.89:0/3680199495":"2024-07-16T06:47:44.086461+0000","172.21.15.89:0/1054475873":"2024-07-16T06:47:44.086461+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-07-15T06:49:07.040 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:49:06 smithi089 bash[31372]: audit 2024-07-15T06:49:06.337540+0000 mon.a (mon.0) 778 : audit [DBG] from='client.? 172.21.15.89:0/329979140' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-07-15T06:49:07.040 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:49:06 smithi089 bash[21793]: audit 2024-07-15T06:49:06.337540+0000 mon.a (mon.0) 778 : audit [DBG] from='client.? 172.21.15.89:0/329979140' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-07-15T06:49:07.042 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-07-15T06:49:07.043 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-07-15T06:49:07.043 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-07-15T06:49:07.043 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-07-15T06:49:07.044 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-07-15T06:49:07.044 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-07-15T06:49:07.045 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-07-15T06:49:07.045 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-07-15T06:49:07.056 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:49:06 smithi134 bash[25019]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:49:06] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-07-15T06:49:07.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:49:06 smithi134 bash[24068]: audit 2024-07-15T06:49:06.337540+0000 mon.a (mon.0) 778 : audit [DBG] from='client.? 172.21.15.89:0/329979140' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-07-15T06:49:08.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:49:07 smithi134 bash[24068]: cluster 2024-07-15T06:49:06.127167+0000 mgr.y (mgr.24272) 74 : cluster [DBG] pgmap v48: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:49:08.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:49:07 smithi089 bash[21793]: cluster 2024-07-15T06:49:06.127167+0000 mgr.y (mgr.24272) 74 : cluster [DBG] pgmap v48: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:49:08.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:49:07 smithi089 bash[31372]: cluster 2024-07-15T06:49:06.127167+0000 mgr.y (mgr.24272) 74 : cluster [DBG] pgmap v48: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:49:10.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:49:09 smithi134 bash[24068]: cluster 2024-07-15T06:49:08.127713+0000 mgr.y (mgr.24272) 75 : cluster [DBG] pgmap v49: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:49:10.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:49:09 smithi089 bash[31372]: cluster 2024-07-15T06:49:08.127713+0000 mgr.y (mgr.24272) 75 : cluster [DBG] pgmap v49: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:49:10.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:49:09 smithi089 bash[21793]: cluster 2024-07-15T06:49:08.127713+0000 mgr.y (mgr.24272) 75 : cluster [DBG] pgmap v49: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:49:11.935 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T06:49:11.935 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T06:49:11.935 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T06:49:11.936 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T06:49:11.938 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T06:49:11.938 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T06:49:11.939 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T06:49:11.939 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T06:49:12.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:49:11 smithi134 bash[24068]: cluster 2024-07-15T06:49:10.128259+0000 mgr.y (mgr.24272) 76 : cluster [DBG] pgmap v50: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:49:12.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:49:11 smithi089 bash[21793]: cluster 2024-07-15T06:49:10.128259+0000 mgr.y (mgr.24272) 76 : cluster [DBG] pgmap v50: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:49:12.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:49:11 smithi089 bash[31372]: cluster 2024-07-15T06:49:10.128259+0000 mgr.y (mgr.24272) 76 : cluster [DBG] pgmap v50: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:49:13.119 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:49:12 smithi089 bash[22045]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:49:12] "GET /metrics HTTP/1.1" 200 203260 "" "Prometheus/2.43.0" 2024-07-15T06:49:14.003 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:49:13 smithi089 bash[21793]: cluster 2024-07-15T06:49:12.128812+0000 mgr.y (mgr.24272) 77 : cluster [DBG] pgmap v51: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:49:14.004 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:49:13 smithi089 bash[31372]: cluster 2024-07-15T06:49:12.128812+0000 mgr.y (mgr.24272) 77 : cluster [DBG] pgmap v51: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:49:14.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:49:13 smithi134 bash[24068]: cluster 2024-07-15T06:49:12.128812+0000 mgr.y (mgr.24272) 77 : cluster [DBG] pgmap v51: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:49:15.024 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:49:14 smithi089 bash[21793]: audit 2024-07-15T06:49:14.189378+0000 mon.a (mon.0) 779 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:49:15.024 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:49:14 smithi089 bash[31372]: audit 2024-07-15T06:49:14.189378+0000 mon.a (mon.0) 779 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:49:15.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:49:14 smithi134 bash[24068]: audit 2024-07-15T06:49:14.189378+0000 mon.a (mon.0) 779 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:49:16.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:49:15 smithi134 bash[24068]: cluster 2024-07-15T06:49:14.129230+0000 mgr.y (mgr.24272) 78 : cluster [DBG] pgmap v52: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:49:16.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:49:15 smithi089 bash[21793]: cluster 2024-07-15T06:49:14.129230+0000 mgr.y (mgr.24272) 78 : cluster [DBG] pgmap v52: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:49:16.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:49:15 smithi089 bash[31372]: cluster 2024-07-15T06:49:14.129230+0000 mgr.y (mgr.24272) 78 : cluster [DBG] pgmap v52: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:49:17.305 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:49:16 smithi134 bash[25019]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:49:16] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-07-15T06:49:18.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:49:17 smithi134 bash[24068]: cluster 2024-07-15T06:49:16.129787+0000 mgr.y (mgr.24272) 79 : cluster [DBG] pgmap v53: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:49:18.078 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:49:17 smithi089 bash[31372]: cluster 2024-07-15T06:49:16.129787+0000 mgr.y (mgr.24272) 79 : cluster [DBG] pgmap v53: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:49:18.078 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:49:17 smithi089 bash[21793]: cluster 2024-07-15T06:49:16.129787+0000 mgr.y (mgr.24272) 79 : cluster [DBG] pgmap v53: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:49:20.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:49:19 smithi134 bash[24068]: cluster 2024-07-15T06:49:18.130183+0000 mgr.y (mgr.24272) 80 : cluster [DBG] pgmap v54: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:49:20.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:49:19 smithi089 bash[21793]: cluster 2024-07-15T06:49:18.130183+0000 mgr.y (mgr.24272) 80 : cluster [DBG] pgmap v54: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:49:20.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:49:19 smithi089 bash[31372]: cluster 2024-07-15T06:49:18.130183+0000 mgr.y (mgr.24272) 80 : cluster [DBG] pgmap v54: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:49:22.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:49:21 smithi134 bash[24068]: cluster 2024-07-15T06:49:20.130741+0000 mgr.y (mgr.24272) 81 : cluster [DBG] pgmap v55: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:49:22.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:49:21 smithi089 bash[21793]: cluster 2024-07-15T06:49:20.130741+0000 mgr.y (mgr.24272) 81 : cluster [DBG] pgmap v55: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:49:22.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:49:21 smithi089 bash[31372]: cluster 2024-07-15T06:49:20.130741+0000 mgr.y (mgr.24272) 81 : cluster [DBG] pgmap v55: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:49:22.916 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:49:22 smithi089 bash[21793]: cluster 2024-07-15T06:49:22.131375+0000 mgr.y (mgr.24272) 82 : cluster [DBG] pgmap v56: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:49:22.917 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:49:22 smithi089 bash[31372]: cluster 2024-07-15T06:49:22.131375+0000 mgr.y (mgr.24272) 82 : cluster [DBG] pgmap v56: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:49:23.017 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:49:22 smithi089 bash[22045]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:49:22] "GET /metrics HTTP/1.1" 200 203231 "" "Prometheus/2.43.0" 2024-07-15T06:49:23.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:49:22 smithi134 bash[24068]: cluster 2024-07-15T06:49:22.131375+0000 mgr.y (mgr.24272) 82 : cluster [DBG] pgmap v56: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:49:25.333 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:49:25 smithi089 bash[21793]: cluster 2024-07-15T06:49:24.131989+0000 mgr.y (mgr.24272) 83 : cluster [DBG] pgmap v57: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:49:25.333 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:49:25 smithi089 bash[31372]: cluster 2024-07-15T06:49:24.131989+0000 mgr.y (mgr.24272) 83 : cluster [DBG] pgmap v57: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:49:25.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:49:25 smithi134 bash[24068]: cluster 2024-07-15T06:49:24.131989+0000 mgr.y (mgr.24272) 83 : cluster [DBG] pgmap v57: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:49:26.414 INFO:teuthology.orchestra.run.smithi089.stdout:107374182476 2024-07-15T06:49:26.414 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-07-15T06:49:27.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:49:27 smithi134 bash[24068]: cluster 2024-07-15T06:49:26.132688+0000 mgr.y (mgr.24272) 84 : cluster [DBG] pgmap v58: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:49:27.305 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:49:26 smithi134 bash[25019]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:49:26] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-07-15T06:49:27.619 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:49:27 smithi089 bash[31372]: cluster 2024-07-15T06:49:26.132688+0000 mgr.y (mgr.24272) 84 : cluster [DBG] pgmap v58: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:49:27.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:49:27 smithi089 bash[21793]: cluster 2024-07-15T06:49:26.132688+0000 mgr.y (mgr.24272) 84 : cluster [DBG] pgmap v58: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:49:27.788 INFO:teuthology.orchestra.run.smithi089.stdout:163208757302 2024-07-15T06:49:27.789 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-07-15T06:49:28.736 INFO:teuthology.orchestra.run.smithi089.stdout:34359738476 2024-07-15T06:49:28.737 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-07-15T06:49:29.132 INFO:teuthology.orchestra.run.smithi089.stdout:77309411414 2024-07-15T06:49:29.132 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-07-15T06:49:29.369 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:49:29 smithi089 bash[31372]: cluster 2024-07-15T06:49:28.133129+0000 mgr.y (mgr.24272) 85 : cluster [DBG] pgmap v59: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:49:29.369 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:49:29 smithi089 bash[31372]: audit 2024-07-15T06:49:29.189760+0000 mon.a (mon.0) 780 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:49:29.369 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:49:29 smithi089 bash[21793]: cluster 2024-07-15T06:49:28.133129+0000 mgr.y (mgr.24272) 85 : cluster [DBG] pgmap v59: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:49:29.369 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:49:29 smithi089 bash[21793]: audit 2024-07-15T06:49:29.189760+0000 mon.a (mon.0) 780 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:49:29.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:49:29 smithi134 bash[24068]: cluster 2024-07-15T06:49:28.133129+0000 mgr.y (mgr.24272) 85 : cluster [DBG] pgmap v59: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:49:29.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:49:29 smithi134 bash[24068]: audit 2024-07-15T06:49:29.189760+0000 mon.a (mon.0) 780 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:49:30.463 INFO:teuthology.orchestra.run.smithi089.stdout:128849018946 2024-07-15T06:49:30.463 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-07-15T06:49:30.567 INFO:teuthology.orchestra.run.smithi089.stdout:55834574945 2024-07-15T06:49:30.568 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-07-15T06:49:30.573 INFO:teuthology.orchestra.run.smithi089.stdout:197568495661 2024-07-15T06:49:30.573 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-07-15T06:49:30.790 INFO:teuthology.orchestra.run.smithi089.stdout:231928234019 2024-07-15T06:49:30.790 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-07-15T06:49:31.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:49:31 smithi134 bash[24068]: cluster 2024-07-15T06:49:30.133718+0000 mgr.y (mgr.24272) 86 : cluster [DBG] pgmap v60: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:49:31.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:49:31 smithi089 bash[21793]: cluster 2024-07-15T06:49:30.133718+0000 mgr.y (mgr.24272) 86 : cluster [DBG] pgmap v60: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:49:31.619 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:49:31 smithi089 bash[31372]: cluster 2024-07-15T06:49:30.133718+0000 mgr.y (mgr.24272) 86 : cluster [DBG] pgmap v60: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:49:33.119 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:49:32 smithi089 bash[22045]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:49:32] "GET /metrics HTTP/1.1" 200 203236 "" "Prometheus/2.43.0" 2024-07-15T06:49:33.225 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T06:49:33.225 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T06:49:33.228 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T06:49:33.229 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T06:49:33.448 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:49:33 smithi089 bash[21793]: cluster 2024-07-15T06:49:32.134323+0000 mgr.y (mgr.24272) 87 : cluster [DBG] pgmap v61: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:49:33.448 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:49:33 smithi089 bash[31372]: cluster 2024-07-15T06:49:32.134323+0000 mgr.y (mgr.24272) 87 : cluster [DBG] pgmap v61: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:49:33.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:49:33 smithi134 bash[24068]: cluster 2024-07-15T06:49:32.134323+0000 mgr.y (mgr.24272) 87 : cluster [DBG] pgmap v61: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:49:35.262 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T06:49:35.262 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T06:49:35.263 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T06:49:35.263 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T06:49:35.446 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:49:35 smithi089 bash[21793]: cluster 2024-07-15T06:49:34.134864+0000 mgr.y (mgr.24272) 88 : cluster [DBG] pgmap v62: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:49:35.446 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:49:35 smithi089 bash[31372]: cluster 2024-07-15T06:49:34.134864+0000 mgr.y (mgr.24272) 88 : cluster [DBG] pgmap v62: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:49:35.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:49:35 smithi134 bash[24068]: cluster 2024-07-15T06:49:34.134864+0000 mgr.y (mgr.24272) 88 : cluster [DBG] pgmap v62: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:49:37.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:49:37 smithi134 bash[24068]: cluster 2024-07-15T06:49:36.135665+0000 mgr.y (mgr.24272) 89 : cluster [DBG] pgmap v63: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:49:37.305 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:49:36 smithi134 bash[25019]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:49:36] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-07-15T06:49:37.369 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:49:37 smithi089 bash[21793]: cluster 2024-07-15T06:49:36.135665+0000 mgr.y (mgr.24272) 89 : cluster [DBG] pgmap v63: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:49:37.369 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:49:37 smithi089 bash[31372]: cluster 2024-07-15T06:49:36.135665+0000 mgr.y (mgr.24272) 89 : cluster [DBG] pgmap v63: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:49:39.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:49:39 smithi134 bash[24068]: cluster 2024-07-15T06:49:38.136196+0000 mgr.y (mgr.24272) 90 : cluster [DBG] pgmap v64: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:49:39.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:49:39 smithi089 bash[21793]: cluster 2024-07-15T06:49:38.136196+0000 mgr.y (mgr.24272) 90 : cluster [DBG] pgmap v64: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:49:39.619 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:49:39 smithi089 bash[31372]: cluster 2024-07-15T06:49:38.136196+0000 mgr.y (mgr.24272) 90 : cluster [DBG] pgmap v64: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:49:41.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:49:41 smithi134 bash[24068]: cluster 2024-07-15T06:49:40.136934+0000 mgr.y (mgr.24272) 91 : cluster [DBG] pgmap v65: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:49:41.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:49:41 smithi089 bash[21793]: cluster 2024-07-15T06:49:40.136934+0000 mgr.y (mgr.24272) 91 : cluster [DBG] pgmap v65: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:49:41.619 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:49:41 smithi089 bash[31372]: cluster 2024-07-15T06:49:40.136934+0000 mgr.y (mgr.24272) 91 : cluster [DBG] pgmap v65: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:49:42.869 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:49:42 smithi089 bash[22045]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:49:42] "GET /metrics HTTP/1.1" 200 203236 "" "Prometheus/2.43.0" 2024-07-15T06:49:42.886 INFO:teuthology.orchestra.run.smithi089.stdout:77309411417 2024-07-15T06:49:43.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:49:43 smithi134 bash[24068]: cluster 2024-07-15T06:49:42.137300+0000 mgr.y (mgr.24272) 92 : cluster [DBG] pgmap v66: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:49:43.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:49:43 smithi134 bash[24068]: audit 2024-07-15T06:49:42.892591+0000 mon.a (mon.0) 781 : audit [DBG] from='client.? 172.21.15.89:0/2484773662' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-07-15T06:49:43.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:49:43 smithi089 bash[21793]: cluster 2024-07-15T06:49:42.137300+0000 mgr.y (mgr.24272) 92 : cluster [DBG] pgmap v66: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:49:43.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:49:43 smithi089 bash[21793]: audit 2024-07-15T06:49:42.892591+0000 mon.a (mon.0) 781 : audit [DBG] from='client.? 172.21.15.89:0/2484773662' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-07-15T06:49:43.619 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:49:43 smithi089 bash[31372]: cluster 2024-07-15T06:49:42.137300+0000 mgr.y (mgr.24272) 92 : cluster [DBG] pgmap v66: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:49:43.619 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:49:43 smithi089 bash[31372]: audit 2024-07-15T06:49:42.892591+0000 mon.a (mon.0) 781 : audit [DBG] from='client.? 172.21.15.89:0/2484773662' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-07-15T06:49:44.526 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:49:44 smithi089 bash[21793]: audit 2024-07-15T06:49:44.190157+0000 mon.a (mon.0) 782 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:49:44.527 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:49:44 smithi089 bash[31372]: audit 2024-07-15T06:49:44.190157+0000 mon.a (mon.0) 782 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:49:44.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:49:44 smithi134 bash[24068]: audit 2024-07-15T06:49:44.190157+0000 mon.a (mon.0) 782 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:49:44.572 INFO:teuthology.orchestra.run.smithi089.stdout:34359738479 2024-07-15T06:49:45.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:49:45 smithi134 bash[24068]: cluster 2024-07-15T06:49:44.137802+0000 mgr.y (mgr.24272) 93 : cluster [DBG] pgmap v67: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:49:45.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:49:45 smithi134 bash[24068]: audit 2024-07-15T06:49:44.578577+0000 mon.a (mon.0) 783 : audit [DBG] from='client.? 172.21.15.89:0/515378870' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-07-15T06:49:45.590 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:49:45 smithi089 bash[21793]: cluster 2024-07-15T06:49:44.137802+0000 mgr.y (mgr.24272) 93 : cluster [DBG] pgmap v67: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:49:45.590 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:49:45 smithi089 bash[21793]: audit 2024-07-15T06:49:44.578577+0000 mon.a (mon.0) 783 : audit [DBG] from='client.? 172.21.15.89:0/515378870' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-07-15T06:49:45.591 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:49:45 smithi089 bash[31372]: cluster 2024-07-15T06:49:44.137802+0000 mgr.y (mgr.24272) 93 : cluster [DBG] pgmap v67: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:49:45.591 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:49:45 smithi089 bash[31372]: audit 2024-07-15T06:49:44.578577+0000 mon.a (mon.0) 783 : audit [DBG] from='client.? 172.21.15.89:0/515378870' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-07-15T06:49:46.322 INFO:teuthology.orchestra.run.smithi089.stdout:107374182481 2024-07-15T06:49:46.805 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411414 got 77309411417 for osd.2 2024-07-15T06:49:46.806 DEBUG:teuthology.parallel:result is None 2024-07-15T06:49:47.215 INFO:teuthology.orchestra.run.smithi089.stdout:163208757307 2024-07-15T06:49:47.229 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:49:46 smithi134 bash[25019]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:49:46] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-07-15T06:49:47.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:49:47 smithi134 bash[24068]: cluster 2024-07-15T06:49:46.138434+0000 mgr.y (mgr.24272) 94 : cluster [DBG] pgmap v68: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:49:47.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:49:47 smithi134 bash[24068]: audit 2024-07-15T06:49:46.327999+0000 mon.c (mon.2) 21 : audit [DBG] from='client.? 172.21.15.89:0/1997580761' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-07-15T06:49:47.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:49:47 smithi134 bash[24068]: audit 2024-07-15T06:49:47.221425+0000 mon.a (mon.0) 784 : audit [DBG] from='client.? 172.21.15.89:0/1226043474' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-07-15T06:49:47.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:49:47 smithi089 bash[21793]: cluster 2024-07-15T06:49:46.138434+0000 mgr.y (mgr.24272) 94 : cluster [DBG] pgmap v68: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:49:47.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:49:47 smithi089 bash[21793]: audit 2024-07-15T06:49:46.327999+0000 mon.c (mon.2) 21 : audit [DBG] from='client.? 172.21.15.89:0/1997580761' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-07-15T06:49:47.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:49:47 smithi089 bash[21793]: audit 2024-07-15T06:49:47.221425+0000 mon.a (mon.0) 784 : audit [DBG] from='client.? 172.21.15.89:0/1226043474' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-07-15T06:49:47.619 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:49:47 smithi089 bash[31372]: cluster 2024-07-15T06:49:46.138434+0000 mgr.y (mgr.24272) 94 : cluster [DBG] pgmap v68: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:49:47.619 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:49:47 smithi089 bash[31372]: audit 2024-07-15T06:49:46.327999+0000 mon.c (mon.2) 21 : audit [DBG] from='client.? 172.21.15.89:0/1997580761' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-07-15T06:49:47.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:49:47 smithi089 bash[31372]: audit 2024-07-15T06:49:47.221425+0000 mon.a (mon.0) 784 : audit [DBG] from='client.? 172.21.15.89:0/1226043474' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-07-15T06:49:48.847 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738476 got 34359738479 for osd.0 2024-07-15T06:49:48.847 DEBUG:teuthology.parallel:result is None 2024-07-15T06:49:49.083 INFO:tasks.cephadm.ceph_manager.ceph:need seq 107374182476 got 107374182481 for osd.3 2024-07-15T06:49:49.084 DEBUG:teuthology.parallel:result is None 2024-07-15T06:49:49.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:49:49 smithi134 bash[24068]: cluster 2024-07-15T06:49:48.139010+0000 mgr.y (mgr.24272) 95 : cluster [DBG] pgmap v69: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:49:49.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:49:49 smithi089 bash[21793]: cluster 2024-07-15T06:49:48.139010+0000 mgr.y (mgr.24272) 95 : cluster [DBG] pgmap v69: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:49:49.619 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:49:49 smithi089 bash[31372]: cluster 2024-07-15T06:49:48.139010+0000 mgr.y (mgr.24272) 95 : cluster [DBG] pgmap v69: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:49:50.062 INFO:tasks.cephadm.ceph_manager.ceph:need seq 163208757302 got 163208757307 for osd.5 2024-07-15T06:49:50.062 DEBUG:teuthology.parallel:result is None 2024-07-15T06:49:50.722 INFO:teuthology.orchestra.run.smithi089.stdout:231928234024 2024-07-15T06:49:50.846 INFO:teuthology.orchestra.run.smithi089.stdout:128849018951 2024-07-15T06:49:51.093 INFO:teuthology.orchestra.run.smithi089.stdout:55834574950 2024-07-15T06:49:51.220 INFO:teuthology.orchestra.run.smithi089.stdout:197568495666 2024-07-15T06:49:51.369 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:49:51 smithi089 bash[21793]: cluster 2024-07-15T06:49:50.139603+0000 mgr.y (mgr.24272) 96 : cluster [DBG] pgmap v70: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:49:51.369 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:49:51 smithi089 bash[21793]: audit 2024-07-15T06:49:50.728554+0000 mon.c (mon.2) 22 : audit [DBG] from='client.? 172.21.15.89:0/1182335846' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-07-15T06:49:51.369 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:49:51 smithi089 bash[21793]: audit 2024-07-15T06:49:50.852254+0000 mon.a (mon.0) 785 : audit [DBG] from='client.? 172.21.15.89:0/3305968353' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-07-15T06:49:51.369 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:49:51 smithi089 bash[21793]: audit 2024-07-15T06:49:51.099249+0000 mon.a (mon.0) 786 : audit [DBG] from='client.? 172.21.15.89:0/1768179823' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-07-15T06:49:51.369 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:49:51 smithi089 bash[21793]: audit 2024-07-15T06:49:51.226443+0000 mon.c (mon.2) 23 : audit [DBG] from='client.? 172.21.15.89:0/1976000652' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-07-15T06:49:51.369 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:49:51 smithi089 bash[31372]: cluster 2024-07-15T06:49:50.139603+0000 mgr.y (mgr.24272) 96 : cluster [DBG] pgmap v70: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:49:51.369 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:49:51 smithi089 bash[31372]: audit 2024-07-15T06:49:50.728554+0000 mon.c (mon.2) 22 : audit [DBG] from='client.? 172.21.15.89:0/1182335846' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-07-15T06:49:51.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:49:51 smithi089 bash[31372]: audit 2024-07-15T06:49:50.852254+0000 mon.a (mon.0) 785 : audit [DBG] from='client.? 172.21.15.89:0/3305968353' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-07-15T06:49:51.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:49:51 smithi089 bash[31372]: audit 2024-07-15T06:49:51.099249+0000 mon.a (mon.0) 786 : audit [DBG] from='client.? 172.21.15.89:0/1768179823' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-07-15T06:49:51.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:49:51 smithi089 bash[31372]: audit 2024-07-15T06:49:51.226443+0000 mon.c (mon.2) 23 : audit [DBG] from='client.? 172.21.15.89:0/1976000652' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-07-15T06:49:51.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:49:51 smithi134 bash[24068]: cluster 2024-07-15T06:49:50.139603+0000 mgr.y (mgr.24272) 96 : cluster [DBG] pgmap v70: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:49:51.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:49:51 smithi134 bash[24068]: audit 2024-07-15T06:49:50.728554+0000 mon.c (mon.2) 22 : audit [DBG] from='client.? 172.21.15.89:0/1182335846' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-07-15T06:49:51.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:49:51 smithi134 bash[24068]: audit 2024-07-15T06:49:50.852254+0000 mon.a (mon.0) 785 : audit [DBG] from='client.? 172.21.15.89:0/3305968353' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-07-15T06:49:51.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:49:51 smithi134 bash[24068]: audit 2024-07-15T06:49:51.099249+0000 mon.a (mon.0) 786 : audit [DBG] from='client.? 172.21.15.89:0/1768179823' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-07-15T06:49:51.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:49:51 smithi134 bash[24068]: audit 2024-07-15T06:49:51.226443+0000 mon.c (mon.2) 23 : audit [DBG] from='client.? 172.21.15.89:0/1976000652' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-07-15T06:49:52.351 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849018946 got 128849018951 for osd.4 2024-07-15T06:49:52.351 DEBUG:teuthology.parallel:result is None 2024-07-15T06:49:52.440 INFO:tasks.cephadm.ceph_manager.ceph:need seq 231928234019 got 231928234024 for osd.7 2024-07-15T06:49:52.440 DEBUG:teuthology.parallel:result is None 2024-07-15T06:49:52.923 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:49:52 smithi089 bash[22045]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:49:52] "GET /metrics HTTP/1.1" 200 203236 "" "Prometheus/2.43.0" 2024-07-15T06:49:52.924 INFO:tasks.cephadm.ceph_manager.ceph:need seq 197568495661 got 197568495666 for osd.6 2024-07-15T06:49:52.924 DEBUG:teuthology.parallel:result is None 2024-07-15T06:49:52.974 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574945 got 55834574950 for osd.1 2024-07-15T06:49:52.974 DEBUG:teuthology.parallel:result is None 2024-07-15T06:49:52.974 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-07-15T06:49:52.975 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph pg dump --format=json 2024-07-15T06:49:53.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:49:53 smithi134 bash[24068]: cluster 2024-07-15T06:49:52.140180+0000 mgr.y (mgr.24272) 97 : cluster [DBG] pgmap v71: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:49:53.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:49:53 smithi089 bash[21793]: cluster 2024-07-15T06:49:52.140180+0000 mgr.y (mgr.24272) 97 : cluster [DBG] pgmap v71: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:49:53.619 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:49:53 smithi089 bash[31372]: cluster 2024-07-15T06:49:52.140180+0000 mgr.y (mgr.24272) 97 : cluster [DBG] pgmap v71: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:49:55.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:49:55 smithi134 bash[24068]: cluster 2024-07-15T06:49:54.140726+0000 mgr.y (mgr.24272) 98 : cluster [DBG] pgmap v72: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:49:55.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:49:55 smithi089 bash[21793]: cluster 2024-07-15T06:49:54.140726+0000 mgr.y (mgr.24272) 98 : cluster [DBG] pgmap v72: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:49:55.619 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:49:55 smithi089 bash[31372]: cluster 2024-07-15T06:49:54.140726+0000 mgr.y (mgr.24272) 98 : cluster [DBG] pgmap v72: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:49:57.248 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:49:56 smithi134 bash[25019]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:49:56] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-07-15T06:49:57.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:49:57 smithi134 bash[24068]: cluster 2024-07-15T06:49:56.141368+0000 mgr.y (mgr.24272) 99 : cluster [DBG] pgmap v73: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:49:57.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:49:57 smithi089 bash[21793]: cluster 2024-07-15T06:49:56.141368+0000 mgr.y (mgr.24272) 99 : cluster [DBG] pgmap v73: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:49:57.619 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:49:57 smithi089 bash[31372]: cluster 2024-07-15T06:49:56.141368+0000 mgr.y (mgr.24272) 99 : cluster [DBG] pgmap v73: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:49:57.746 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T06:49:59.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:49:59 smithi134 bash[24068]: cluster 2024-07-15T06:49:58.141914+0000 mgr.y (mgr.24272) 100 : cluster [DBG] pgmap v74: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:49:59.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:49:59 smithi134 bash[24068]: audit 2024-07-15T06:49:59.190633+0000 mon.a (mon.0) 787 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:49:59.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:49:59 smithi089 bash[21793]: cluster 2024-07-15T06:49:58.141914+0000 mgr.y (mgr.24272) 100 : cluster [DBG] pgmap v74: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:49:59.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:49:59 smithi089 bash[21793]: audit 2024-07-15T06:49:59.190633+0000 mon.a (mon.0) 787 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:49:59.619 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:49:59 smithi089 bash[31372]: cluster 2024-07-15T06:49:58.141914+0000 mgr.y (mgr.24272) 100 : cluster [DBG] pgmap v74: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:49:59.619 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:49:59 smithi089 bash[31372]: audit 2024-07-15T06:49:59.190633+0000 mon.a (mon.0) 787 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:50:00.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:00 smithi134 bash[24068]: cluster 2024-07-15T06:50:00.000195+0000 mon.a (mon.0) 788 : cluster [INF] overall HEALTH_OK 2024-07-15T06:50:00.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:00 smithi089 bash[21793]: cluster 2024-07-15T06:50:00.000195+0000 mon.a (mon.0) 788 : cluster [INF] overall HEALTH_OK 2024-07-15T06:50:00.619 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:00 smithi089 bash[31372]: cluster 2024-07-15T06:50:00.000195+0000 mon.a (mon.0) 788 : cluster [INF] overall HEALTH_OK 2024-07-15T06:50:01.119 INFO:teuthology.orchestra.run.smithi089.stderr:dumped all 2024-07-15T06:50:01.120 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-07-15T06:50:01.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:01 smithi134 bash[24068]: cluster 2024-07-15T06:50:00.142475+0000 mgr.y (mgr.24272) 101 : cluster [DBG] pgmap v75: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:50:01.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:01 smithi089 bash[21793]: cluster 2024-07-15T06:50:00.142475+0000 mgr.y (mgr.24272) 101 : cluster [DBG] pgmap v75: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:50:01.619 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:01 smithi089 bash[31372]: cluster 2024-07-15T06:50:00.142475+0000 mgr.y (mgr.24272) 101 : cluster [DBG] pgmap v75: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:50:01.798 INFO:teuthology.orchestra.run.smithi089.stdout:{"pg_ready":true,"pg_map":{"version":75,"stamp":"2024-07-15T06:50:00.142144+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2385508,"kb_used_data":6660,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747575708,"statfs":{"total":767960285184,"available":765517524992,"internally_reserved":0,"allocated":6819840,"data_stored":3876242,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003395"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2024-07-15T06:48:29.972552+0000","last_change":"2024-07-15T06:48:28.675814+0000","last_active":"2024-07-15T06:48:29.972552+0000","last_peered":"2024-07-15T06:48:29.972552+0000","last_clean":"2024-07-15T06:48:29.972552+0000","last_became_active":"2024-07-15T06:48:28.674798+0000","last_became_peered":"2024-07-15T06:48:28.674798+0000","last_unstale":"2024-07-15T06:48:29.972552+0000","last_undegraded":"2024-07-15T06:48:29.972552+0000","last_fullsized":"2024-07-15T06:48:29.972552+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_clean_scrub_stamp":"2024-07-15T06:48:27.655327+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T14:36:19.382840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2024-07-15T06:48:30.120901+0000","last_change":"2024-07-15T06:48:28.872731+0000","last_active":"2024-07-15T06:48:30.120901+0000","last_peered":"2024-07-15T06:48:30.120901+0000","last_clean":"2024-07-15T06:48:30.120901+0000","last_became_active":"2024-07-15T06:48:28.871744+0000","last_became_peered":"2024-07-15T06:48:28.871744+0000","last_unstale":"2024-07-15T06:48:30.120901+0000","last_undegraded":"2024-07-15T06:48:30.120901+0000","last_fullsized":"2024-07-15T06:48:30.120901+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_clean_scrub_stamp":"2024-07-15T06:48:27.655327+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T11:36:09.487214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2024-07-15T06:48:29.669133+0000","last_change":"2024-07-15T06:48:28.872322+0000","last_active":"2024-07-15T06:48:29.669133+0000","last_peered":"2024-07-15T06:48:29.669133+0000","last_clean":"2024-07-15T06:48:29.669133+0000","last_became_active":"2024-07-15T06:48:28.871320+0000","last_became_peered":"2024-07-15T06:48:28.871320+0000","last_unstale":"2024-07-15T06:48:29.669133+0000","last_undegraded":"2024-07-15T06:48:29.669133+0000","last_fullsized":"2024-07-15T06:48:29.669133+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_clean_scrub_stamp":"2024-07-15T06:48:27.655327+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T14:24:12.762681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2024-07-15T06:48:30.120812+0000","last_change":"2024-07-15T06:48:28.738368+0000","last_active":"2024-07-15T06:48:30.120812+0000","last_peered":"2024-07-15T06:48:30.120812+0000","last_clean":"2024-07-15T06:48:30.120812+0000","last_became_active":"2024-07-15T06:48:28.737382+0000","last_became_peered":"2024-07-15T06:48:28.737382+0000","last_unstale":"2024-07-15T06:48:30.120812+0000","last_undegraded":"2024-07-15T06:48:30.120812+0000","last_fullsized":"2024-07-15T06:48:30.120812+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_clean_scrub_stamp":"2024-07-15T06:48:27.655327+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T17:01:42.558382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2024-07-15T06:48:30.405779+0000","last_change":"2024-07-15T06:48:28.674943+0000","last_active":"2024-07-15T06:48:30.405779+0000","last_peered":"2024-07-15T06:48:30.405779+0000","last_clean":"2024-07-15T06:48:30.405779+0000","last_became_active":"2024-07-15T06:48:28.674174+0000","last_became_peered":"2024-07-15T06:48:28.674174+0000","last_unstale":"2024-07-15T06:48:30.405779+0000","last_undegraded":"2024-07-15T06:48:30.405779+0000","last_fullsized":"2024-07-15T06:48:30.405779+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_clean_scrub_stamp":"2024-07-15T06:48:27.655327+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T17:40:56.394801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2024-07-15T06:48:29.665606+0000","last_change":"2024-07-15T06:48:28.738441+0000","last_active":"2024-07-15T06:48:29.665606+0000","last_peered":"2024-07-15T06:48:29.665606+0000","last_clean":"2024-07-15T06:48:29.665606+0000","last_became_active":"2024-07-15T06:48:28.737053+0000","last_became_peered":"2024-07-15T06:48:28.737053+0000","last_unstale":"2024-07-15T06:48:29.665606+0000","last_undegraded":"2024-07-15T06:48:29.665606+0000","last_fullsized":"2024-07-15T06:48:29.665606+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_clean_scrub_stamp":"2024-07-15T06:48:27.655327+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T08:33:33.092342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2024-07-15T06:48:29.669009+0000","last_change":"2024-07-15T06:48:28.737091+0000","last_active":"2024-07-15T06:48:29.669009+0000","last_peered":"2024-07-15T06:48:29.669009+0000","last_clean":"2024-07-15T06:48:29.669009+0000","last_became_active":"2024-07-15T06:48:28.736574+0000","last_became_peered":"2024-07-15T06:48:28.736574+0000","last_unstale":"2024-07-15T06:48:29.669009+0000","last_undegraded":"2024-07-15T06:48:29.669009+0000","last_fullsized":"2024-07-15T06:48:29.669009+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_clean_scrub_stamp":"2024-07-15T06:48:27.655327+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T09:31:07.195677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2024-07-15T06:48:30.405734+0000","last_change":"2024-07-15T06:48:28.672045+0000","last_active":"2024-07-15T06:48:30.405734+0000","last_peered":"2024-07-15T06:48:30.405734+0000","last_clean":"2024-07-15T06:48:30.405734+0000","last_became_active":"2024-07-15T06:48:28.671290+0000","last_became_peered":"2024-07-15T06:48:28.671290+0000","last_unstale":"2024-07-15T06:48:30.405734+0000","last_undegraded":"2024-07-15T06:48:30.405734+0000","last_fullsized":"2024-07-15T06:48:30.405734+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_clean_scrub_stamp":"2024-07-15T06:48:27.655327+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T11:10:59.552676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"59'128","reported_seq":270,"reported_epoch":62,"state":"active+clean","last_fresh":"2024-07-15T06:48:29.669027+0000","last_change":"2024-07-15T06:46:49.356522+0000","last_active":"2024-07-15T06:48:29.669027+0000","last_peered":"2024-07-15T06:48:29.669027+0000","last_clean":"2024-07-15T06:48:29.669027+0000","last_became_active":"2024-07-15T06:46:49.355501+0000","last_became_peered":"2024-07-15T06:46:49.355501+0000","last_unstale":"2024-07-15T06:48:29.669027+0000","last_undegraded":"2024-07-15T06:48:29.669027+0000","last_fullsized":"2024-07-15T06:48:29.669027+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:42:21.582843+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:42:21.582843+0000","last_clean_scrub_stamp":"2024-07-15T06:42:21.582843+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T10:17:36.460676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234026,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298548,"kb_used_data":1192,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446604,"statfs":{"total":95995035648,"available":95689322496,"internally_reserved":0,"allocated":1220608,"data_stored":853268,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Jul 15 06:49:52 2024","interfaces":[{"interface":"back","average":{"1min":0.89100000000000001,"5min":0.92400000000000004,"15min":0.93300000000000005},"min":{"1min":0.49099999999999999,"5min":0.443,"15min":0.443},"max":{"1min":1.532,"5min":1.532,"15min":1.532},"last":0.64600000000000002},{"interface":"front","average":{"1min":0.86199999999999999,"5min":0.879,"15min":0.89300000000000002},"min":{"1min":0.52300000000000002,"5min":0.46600000000000003,"15min":0.46600000000000003},"max":{"1min":1.2609999999999999,"5min":1.2609999999999999,"15min":1.2609999999999999},"last":1.0209999999999999}]},{"osd":1,"last update":"Mon Jul 15 06:49:52 2024","interfaces":[{"interface":"back","average":{"1min":0.84199999999999997,"5min":0.92000000000000004,"15min":0.93500000000000005},"min":{"1min":0.52200000000000002,"5min":0.48499999999999999,"15min":0.48499999999999999},"max":{"1min":1.6579999999999999,"5min":1.6579999999999999,"15min":1.6579999999999999},"last":1.252},{"interface":"front","average":{"1min":0.91100000000000003,"5min":0.96899999999999997,"15min":0.999},"min":{"1min":0.51500000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"max":{"1min":1.3069999999999999,"5min":1.357,"15min":1.357},"last":0.77700000000000002}]},{"osd":2,"last update":"Mon Jul 15 06:49:52 2024","interfaces":[{"interface":"back","average":{"1min":0.92600000000000005,"5min":0.92900000000000005,"15min":0.91200000000000003},"min":{"1min":0.67800000000000005,"5min":0.50900000000000001,"15min":0.50900000000000001},"max":{"1min":1.6279999999999999,"5min":1.6279999999999999,"15min":1.6279999999999999},"last":0.99299999999999999},{"interface":"front","average":{"1min":0.91700000000000004,"5min":0.96799999999999997,"15min":0.97999999999999998},"min":{"1min":0.59299999999999997,"5min":0.55800000000000005,"15min":0.55800000000000005},"max":{"1min":1.802,"5min":1.802,"15min":1.802},"last":0.80700000000000005}]},{"osd":3,"last update":"Mon Jul 15 06:49:52 2024","interfaces":[{"interface":"back","average":{"1min":0.92100000000000004,"5min":0.95099999999999996,"15min":0.94599999999999995},"min":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"max":{"1min":1.704,"5min":1.704,"15min":1.704},"last":0.85199999999999998},{"interface":"front","average":{"1min":1.0089999999999999,"5min":1.0409999999999999,"15min":1.0620000000000001},"min":{"1min":0.75800000000000001,"5min":0.60499999999999998,"15min":0.60499999999999998},"max":{"1min":1.74,"5min":1.74,"15min":1.74},"last":1.1679999999999999}]},{"osd":4,"last update":"Mon Jul 15 06:49:52 2024","interfaces":[{"interface":"back","average":{"1min":0.97499999999999998,"5min":0.96399999999999997,"15min":0.97099999999999997},"min":{"1min":0.59699999999999998,"5min":0.52500000000000002,"15min":0.52500000000000002},"max":{"1min":1.2889999999999999,"5min":1.456,"15min":1.456},"last":0.89900000000000002},{"interface":"front","average":{"1min":1.016,"5min":1.0389999999999999,"15min":1.0620000000000001},"min":{"1min":0.64000000000000001,"5min":0.57099999999999995,"15min":0.57099999999999995},"max":{"1min":2.0630000000000002,"5min":2.0630000000000002,"15min":2.0630000000000002},"last":0.93400000000000005}]},{"osd":5,"last update":"Mon Jul 15 06:49:52 2024","interfaces":[{"interface":"back","average":{"1min":0.97899999999999998,"5min":1.0189999999999999,"15min":1.0469999999999999},"min":{"1min":0.63400000000000001,"5min":0.499,"15min":0.499},"max":{"1min":1.8680000000000001,"5min":1.8680000000000001,"15min":1.8680000000000001},"last":1.0600000000000001},{"interface":"front","average":{"1min":1.073,"5min":1.056,"15min":1.077},"min":{"1min":0.78500000000000003,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":1.7729999999999999,"5min":1.7729999999999999,"15min":1.7729999999999999},"last":1.22}]},{"osd":6,"last update":"Mon Jul 15 06:49:52 2024","interfaces":[{"interface":"back","average":{"1min":1.125,"5min":1.1000000000000001,"15min":1.1240000000000001},"min":{"1min":0.80500000000000005,"5min":0.50600000000000001,"15min":0.50600000000000001},"max":{"1min":1.9079999999999999,"5min":1.9079999999999999,"15min":1.9079999999999999},"last":1.131},{"interface":"front","average":{"1min":1.107,"5min":1.111,"15min":1.131},"min":{"1min":0.71599999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"max":{"1min":1.823,"5min":1.823,"15min":1.823},"last":1.099}]}]},{"osd":6,"up_from":46,"seq":197568495668,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298552,"kb_used_data":1196,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446600,"statfs":{"total":95995035648,"available":95689318400,"internally_reserved":0,"allocated":1224704,"data_stored":853591,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Jul 15 06:49:57 2024","interfaces":[{"interface":"back","average":{"1min":0.91600000000000004,"5min":0.88800000000000001,"15min":0.91800000000000004},"min":{"1min":0.52500000000000002,"5min":0.50700000000000001,"15min":0.50700000000000001},"max":{"1min":1.181,"5min":2.1579999999999999,"15min":2.1579999999999999},"last":0.70199999999999996},{"interface":"front","average":{"1min":0.91600000000000004,"5min":0.88200000000000001,"15min":0.89200000000000002},"min":{"1min":0.627,"5min":0.47099999999999997,"15min":0.47099999999999997},"max":{"1min":1.276,"5min":1.3089999999999999,"15min":1.3089999999999999},"last":0.82799999999999996}]},{"osd":1,"last update":"Mon Jul 15 06:49:57 2024","interfaces":[{"interface":"back","average":{"1min":0.97599999999999998,"5min":0.91700000000000004,"15min":0.91500000000000004},"min":{"1min":0.71999999999999997,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":1.4019999999999999,"5min":1.444,"15min":1.444},"last":0.752},{"interface":"front","average":{"1min":0.89000000000000001,"5min":0.91100000000000003,"15min":0.95699999999999996},"min":{"1min":0.60999999999999999,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":1.4119999999999999,"5min":2.2210000000000001,"15min":2.2210000000000001},"last":0.78500000000000003}]},{"osd":2,"last update":"Mon Jul 15 06:49:57 2024","interfaces":[{"interface":"back","average":{"1min":0.84299999999999997,"5min":0.91400000000000003,"15min":0.97699999999999998},"min":{"1min":0.67700000000000005,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":1.139,"5min":2.754,"15min":2.754},"last":0.872},{"interface":"front","average":{"1min":1,"5min":0.96199999999999997,"15min":1.004},"min":{"1min":0.73999999999999999,"5min":0.501,"15min":0.501},"max":{"1min":1.266,"5min":2.1019999999999999,"15min":2.1019999999999999},"last":1.0289999999999999}]},{"osd":3,"last update":"Mon Jul 15 06:49:57 2024","interfaces":[{"interface":"back","average":{"1min":0.97399999999999998,"5min":0.91200000000000003,"15min":0.90600000000000003},"min":{"1min":0.73399999999999999,"5min":0.498,"15min":0.498},"max":{"1min":1.224,"5min":1.3799999999999999,"15min":1.3799999999999999},"last":0.90500000000000003},{"interface":"front","average":{"1min":0.999,"5min":0.95299999999999996,"15min":0.997},"min":{"1min":0.79600000000000004,"5min":0.52500000000000002,"15min":0.52500000000000002},"max":{"1min":1.321,"5min":2.0600000000000001,"15min":2.0600000000000001},"last":0.94199999999999995}]},{"osd":4,"last update":"Mon Jul 15 06:49:57 2024","interfaces":[{"interface":"back","average":{"1min":0.90300000000000002,"5min":0.79700000000000004,"15min":0.78700000000000003},"min":{"1min":0.624,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":1.165,"5min":1.6120000000000001,"15min":1.6120000000000001},"last":0.64400000000000002},{"interface":"front","average":{"1min":1.004,"5min":0.88500000000000001,"15min":0.85699999999999998},"min":{"1min":0.67500000000000004,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":1.417,"5min":1.417,"15min":1.417},"last":0.98999999999999999}]},{"osd":5,"last update":"Mon Jul 15 06:49:57 2024","interfaces":[{"interface":"back","average":{"1min":1.0329999999999999,"5min":0.89100000000000001,"15min":0.86899999999999999},"min":{"1min":0.72999999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":1.256,"5min":1.6779999999999999,"15min":1.6779999999999999},"last":1.0780000000000001},{"interface":"front","average":{"1min":1.1240000000000001,"5min":0.92000000000000004,"15min":0.873},"min":{"1min":0.78300000000000003,"5min":0.373,"15min":0.373},"max":{"1min":1.3520000000000001,"5min":1.375,"15min":1.375},"last":1.0980000000000001}]},{"osd":7,"last update":"Mon Jul 15 06:49:48 2024","interfaces":[{"interface":"back","average":{"1min":1.097,"5min":0.998,"15min":0.98899999999999999},"min":{"1min":0.53400000000000003,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":1.373,"5min":1.7430000000000001,"15min":1.7430000000000001},"last":0.97499999999999998},{"interface":"front","average":{"1min":1.0609999999999999,"5min":1.004,"15min":0.995},"min":{"1min":0.624,"5min":0.435,"15min":0.435},"max":{"1min":1.4530000000000001,"5min":1.8540000000000001,"15min":1.8540000000000001},"last":1.0449999999999999}]}]},{"osd":5,"up_from":38,"seq":163208757310,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297968,"kb_used_data":612,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447184,"statfs":{"total":95995035648,"available":95689916416,"internally_reserved":0,"allocated":626688,"data_stored":262900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Jul 15 06:49:06 2024","interfaces":[{"interface":"back","average":{"1min":0.88300000000000001,"5min":0.92600000000000005,"15min":0.92200000000000004},"min":{"1min":0.48199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"max":{"1min":1.135,"5min":1.1859999999999999,"15min":1.1859999999999999},"last":1.1259999999999999},{"interface":"front","average":{"1min":0.91700000000000004,"5min":0.89900000000000002,"15min":0.88900000000000001},"min":{"1min":0.55600000000000005,"5min":0.502,"15min":0.502},"max":{"1min":1.177,"5min":1.2210000000000001,"15min":1.2210000000000001},"last":1.1559999999999999}]},{"osd":1,"last update":"Mon Jul 15 06:49:06 2024","interfaces":[{"interface":"back","average":{"1min":0.86199999999999999,"5min":0.83699999999999997,"15min":0.81100000000000005},"min":{"1min":0.54500000000000004,"5min":0.53800000000000003,"15min":0.53800000000000003},"max":{"1min":1.1850000000000001,"5min":1.1850000000000001,"15min":1.1850000000000001},"last":0.82399999999999995},{"interface":"front","average":{"1min":0.86299999999999999,"5min":0.873,"15min":0.88300000000000001},"min":{"1min":0.52500000000000002,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":1.208,"5min":1.268,"15min":1.268},"last":1.073}]},{"osd":2,"last update":"Mon Jul 15 06:49:06 2024","interfaces":[{"interface":"back","average":{"1min":0.89700000000000002,"5min":0.84199999999999997,"15min":0.81699999999999995},"min":{"1min":0.54600000000000004,"5min":0.52500000000000002,"15min":0.52500000000000002},"max":{"1min":1.2869999999999999,"5min":1.2869999999999999,"15min":1.2869999999999999},"last":0.78500000000000003},{"interface":"front","average":{"1min":0.86599999999999999,"5min":0.89700000000000002,"15min":0.91300000000000003},"min":{"1min":0.68799999999999994,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":1.1060000000000001,"5min":1.286,"15min":1.286},"last":1.006}]},{"osd":3,"last update":"Mon Jul 15 06:49:06 2024","interfaces":[{"interface":"back","average":{"1min":0.90300000000000002,"5min":0.85999999999999999,"15min":0.84599999999999997},"min":{"1min":0.61399999999999999,"5min":0.56399999999999995,"15min":0.56399999999999995},"max":{"1min":1.2709999999999999,"5min":1.2709999999999999,"15min":1.2709999999999999},"last":1.0960000000000001},{"interface":"front","average":{"1min":0.872,"5min":0.93400000000000005,"15min":0.94199999999999995},"min":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"max":{"1min":1.23,"5min":1.3320000000000001,"15min":1.3320000000000001},"last":0.89900000000000002}]},{"osd":4,"last update":"Mon Jul 15 06:49:06 2024","interfaces":[{"interface":"back","average":{"1min":0.79900000000000004,"5min":0.748,"15min":0.72599999999999998},"min":{"1min":0.36799999999999999,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":1.091,"5min":1.1910000000000001,"15min":1.1910000000000001},"last":0.85499999999999998},{"interface":"front","average":{"1min":0.85199999999999998,"5min":0.79500000000000004,"15min":0.75900000000000001},"min":{"1min":0.41999999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":1.3420000000000001,"5min":1.3420000000000001,"15min":1.3420000000000001},"last":1.0409999999999999}]},{"osd":6,"last update":"Mon Jul 15 06:49:55 2024","interfaces":[{"interface":"back","average":{"1min":0.99199999999999999,"5min":0.89500000000000002,"15min":0.89600000000000002},"min":{"1min":0.68200000000000005,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":1.2250000000000001,"5min":1.256,"15min":1.256},"last":0.74199999999999999},{"interface":"front","average":{"1min":0.97999999999999998,"5min":0.91900000000000004,"15min":0.91600000000000004},"min":{"1min":0.69399999999999995,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":1.216,"5min":1.3049999999999999,"15min":1.3049999999999999},"last":0.96899999999999997}]},{"osd":7,"last update":"Mon Jul 15 06:49:47 2024","interfaces":[{"interface":"back","average":{"1min":1.0449999999999999,"5min":0.98899999999999999,"15min":0.97699999999999998},"min":{"1min":0.68899999999999995,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":1.256,"5min":1.3380000000000001,"15min":1.3380000000000001},"last":1.2090000000000001},{"interface":"front","average":{"1min":1.022,"5min":0.98099999999999998,"15min":0.98299999999999998},"min":{"1min":0.77500000000000002,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":1.202,"5min":1.3939999999999999,"15min":1.3939999999999999},"last":0.92400000000000004}]}]},{"osd":4,"up_from":30,"seq":128849018953,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297972,"kb_used_data":616,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447180,"statfs":{"total":95995035648,"available":95689912320,"internally_reserved":0,"allocated":630784,"data_stored":263223,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Jul 15 06:49:13 2024","interfaces":[{"interface":"back","average":{"1min":0.84399999999999997,"5min":0.84699999999999998,"15min":0.82699999999999996},"min":{"1min":0.58299999999999996,"5min":0.46600000000000003,"15min":0.46600000000000003},"max":{"1min":1.381,"5min":1.381,"15min":1.381},"last":0.79500000000000004},{"interface":"front","average":{"1min":0.93100000000000005,"5min":0.88600000000000001,"15min":0.83299999999999996},"min":{"1min":0.53400000000000003,"5min":0.49099999999999999,"15min":0.49099999999999999},"max":{"1min":1.381,"5min":1.381,"15min":1.381},"last":1.2110000000000001}]},{"osd":1,"last update":"Mon Jul 15 06:49:13 2024","interfaces":[{"interface":"back","average":{"1min":0.93200000000000005,"5min":0.90300000000000002,"15min":0.86099999999999999},"min":{"1min":0.64000000000000001,"5min":0.497,"15min":0.497},"max":{"1min":1.3120000000000001,"5min":1.3120000000000001,"15min":1.3120000000000001},"last":1.0049999999999999},{"interface":"front","average":{"1min":0.83299999999999996,"5min":0.84199999999999997,"15min":0.86499999999999999},"min":{"1min":0.54300000000000004,"5min":0.501,"15min":0.501},"max":{"1min":1.0960000000000001,"5min":1.3260000000000001,"15min":1.3260000000000001},"last":1.0249999999999999}]},{"osd":2,"last update":"Mon Jul 15 06:49:13 2024","interfaces":[{"interface":"back","average":{"1min":0.92600000000000005,"5min":0.873,"15min":0.83299999999999996},"min":{"1min":0.64400000000000002,"5min":0.49399999999999999,"15min":0.49399999999999999},"max":{"1min":1.204,"5min":1.204,"15min":1.204},"last":1.2310000000000001},{"interface":"front","average":{"1min":0.84199999999999997,"5min":0.86399999999999999,"15min":0.872},"min":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"max":{"1min":1.9910000000000001,"5min":1.9910000000000001,"15min":1.9910000000000001},"last":1.1870000000000001}]},{"osd":3,"last update":"Mon Jul 15 06:49:13 2024","interfaces":[{"interface":"back","average":{"1min":0.98499999999999999,"5min":0.89500000000000002,"15min":0.86599999999999999},"min":{"1min":0.69799999999999995,"5min":0.56000000000000005,"15min":0.56000000000000005},"max":{"1min":2.1709999999999998,"5min":2.1709999999999998,"15min":2.1709999999999998},"last":0.96699999999999997},{"interface":"front","average":{"1min":0.95899999999999996,"5min":0.91500000000000004,"15min":0.90000000000000002},"min":{"1min":0.51400000000000001,"5min":0.48999999999999999,"15min":0.48999999999999999},"max":{"1min":1.4970000000000001,"5min":1.4970000000000001,"15min":1.4970000000000001},"last":1.151}]},{"osd":5,"last update":"Mon Jul 15 06:49:07 2024","interfaces":[{"interface":"back","average":{"1min":0.94599999999999995,"5min":0.78300000000000003,"15min":0.72799999999999998},"min":{"1min":0.40200000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":2.085,"5min":2.085,"15min":2.085},"last":1.089},{"interface":"front","average":{"1min":0.90000000000000002,"5min":0.751,"15min":0.71499999999999997},"min":{"1min":0.32200000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":1.2729999999999999,"5min":1.2729999999999999,"15min":1.2729999999999999},"last":0.88600000000000001}]},{"osd":6,"last update":"Mon Jul 15 06:49:58 2024","interfaces":[{"interface":"back","average":{"1min":0.97999999999999998,"5min":0.85099999999999998,"15min":0.76000000000000001},"min":{"1min":0.42399999999999999,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":1.165,"5min":1.5760000000000001,"15min":1.5760000000000001},"last":0.93000000000000005},{"interface":"front","average":{"1min":1.0029999999999999,"5min":0.87,"15min":0.78800000000000003},"min":{"1min":0.47699999999999998,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":1.246,"5min":1.347,"15min":1.347},"last":1.1220000000000001}]},{"osd":7,"last update":"Mon Jul 15 06:49:48 2024","interfaces":[{"interface":"back","average":{"1min":1.0149999999999999,"5min":0.95399999999999996,"15min":0.92800000000000005},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":2.2469999999999999,"5min":2.2469999999999999,"15min":2.2469999999999999},"last":1.256},{"interface":"front","average":{"1min":1.0369999999999999,"5min":1.0629999999999999,"15min":1.0629999999999999},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":1.653,"5min":1.653,"15min":1.653},"last":1.0629999999999999}]}]},{"osd":3,"up_from":25,"seq":107374182484,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297972,"kb_used_data":616,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447180,"statfs":{"total":95995035648,"available":95689912320,"internally_reserved":0,"allocated":630784,"data_stored":263223,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Jul 15 06:49:22 2024","interfaces":[{"interface":"back","average":{"1min":0.64600000000000002,"5min":0.67100000000000004,"15min":0.64400000000000002},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.90300000000000002,"5min":1.1220000000000001,"15min":1.1220000000000001},"last":0.754},{"interface":"front","average":{"1min":0.73199999999999998,"5min":0.72399999999999998,"15min":0.69999999999999996},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":1.1060000000000001,"5min":1.647,"15min":1.647},"last":0.64600000000000002}]},{"osd":1,"last update":"Mon Jul 15 06:49:22 2024","interfaces":[{"interface":"back","average":{"1min":0.70099999999999996,"5min":0.75600000000000001,"15min":0.70799999999999996},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":1.1739999999999999,"5min":1.3580000000000001,"15min":1.3580000000000001},"last":0.73399999999999999},{"interface":"front","average":{"1min":0.75,"5min":0.73599999999999999,"15min":0.69799999999999995},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":1.0149999999999999,"5min":1.2190000000000001,"15min":1.2190000000000001},"last":0.60799999999999998}]},{"osd":2,"last update":"Mon Jul 15 06:49:22 2024","interfaces":[{"interface":"back","average":{"1min":0.77400000000000002,"5min":0.79300000000000004,"15min":0.753},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":1.157,"5min":1.157,"15min":1.157},"last":0.67900000000000005},{"interface":"front","average":{"1min":0.73899999999999999,"5min":0.76600000000000001,"15min":0.72299999999999998},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":1.3240000000000001,"5min":1.3240000000000001,"15min":1.3240000000000001},"last":0.57299999999999995}]},{"osd":4,"last update":"Mon Jul 15 06:49:17 2024","interfaces":[{"interface":"back","average":{"1min":0.92200000000000004,"5min":0.91800000000000004,"15min":0.876},"min":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"max":{"1min":1.179,"5min":1.2949999999999999,"15min":1.2949999999999999},"last":0.82799999999999996},{"interface":"front","average":{"1min":0.94999999999999996,"5min":0.94999999999999996,"15min":0.90300000000000002},"min":{"1min":0.72399999999999998,"5min":0.60899999999999999,"15min":0.60899999999999999},"max":{"1min":1.538,"5min":1.8140000000000001,"15min":1.8140000000000001},"last":0.83899999999999997}]},{"osd":5,"last update":"Mon Jul 15 06:49:11 2024","interfaces":[{"interface":"back","average":{"1min":0.97599999999999998,"5min":0.97199999999999998,"15min":0.96199999999999997},"min":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"max":{"1min":1.48,"5min":1.7609999999999999,"15min":1.7609999999999999},"last":0.873},{"interface":"front","average":{"1min":0.96799999999999997,"5min":0.98099999999999998,"15min":0.98599999999999999},"min":{"1min":0.70999999999999996,"5min":0.68100000000000005,"15min":0.68100000000000005},"max":{"1min":1.4019999999999999,"5min":1.5069999999999999,"15min":1.5069999999999999},"last":0.70299999999999996}]},{"osd":6,"last update":"Mon Jul 15 06:49:02 2024","interfaces":[{"interface":"back","average":{"1min":1.016,"5min":1.0620000000000001,"15min":1.0840000000000001},"min":{"1min":0.65600000000000003,"5min":0.64500000000000002,"15min":0.64500000000000002},"max":{"1min":1.5880000000000001,"5min":1.8899999999999999,"15min":1.8899999999999999},"last":0.93700000000000006},{"interface":"front","average":{"1min":0.97099999999999997,"5min":1.006,"15min":1.016},"min":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"max":{"1min":1.3859999999999999,"5min":1.3859999999999999,"15min":1.3859999999999999},"last":0.85999999999999999}]},{"osd":7,"last update":"Mon Jul 15 06:49:54 2024","interfaces":[{"interface":"back","average":{"1min":0.94599999999999995,"5min":1.016,"15min":1.0209999999999999},"min":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"max":{"1min":1.155,"5min":1.4730000000000001,"15min":1.4730000000000001},"last":0.76900000000000002},{"interface":"front","average":{"1min":0.98099999999999998,"5min":1.0680000000000001,"15min":1.0940000000000001},"min":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"max":{"1min":1.2589999999999999,"5min":1.6100000000000001,"15min":1.6100000000000001},"last":0.85199999999999998}]}]},{"osd":2,"up_from":18,"seq":77309411421,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297972,"kb_used_data":616,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447180,"statfs":{"total":95995035648,"available":95689912320,"internally_reserved":0,"allocated":630784,"data_stored":263223,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Jul 15 06:49:39 2024","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.63900000000000001,"15min":0.58899999999999997},"min":{"1min":0.28499999999999998,"5min":0.28299999999999997,"15min":0.22700000000000001},"max":{"1min":0.93500000000000005,"5min":1.4690000000000001,"15min":1.4690000000000001},"last":0.61799999999999999},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.68200000000000005,"15min":0.59699999999999998},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.30099999999999999},"max":{"1min":0.95499999999999996,"5min":1.095,"15min":1.095},"last":0.748}]},{"osd":1,"last update":"Mon Jul 15 06:49:39 2024","interfaces":[{"interface":"back","average":{"1min":0.64500000000000002,"5min":0.70499999999999996,"15min":0.64400000000000002},"min":{"1min":0.34699999999999998,"5min":0.31900000000000001,"15min":0.28399999999999997},"max":{"1min":0.85799999999999998,"5min":1.177,"15min":1.177},"last":0.69399999999999995},{"interface":"front","average":{"1min":0.73099999999999998,"5min":0.76100000000000001,"15min":0.65100000000000002},"min":{"1min":0.40699999999999997,"5min":0.314,"15min":0.27400000000000002},"max":{"1min":1.224,"5min":1.2330000000000001,"15min":1.2330000000000001},"last":0.94899999999999995}]},{"osd":3,"last update":"Mon Jul 15 06:49:28 2024","interfaces":[{"interface":"back","average":{"1min":0.70999999999999996,"5min":0.76300000000000001,"15min":0.75600000000000001},"min":{"1min":0.375,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":1.03,"5min":1.256,"15min":1.256},"last":0.80100000000000005},{"interface":"front","average":{"1min":0.78500000000000003,"5min":0.81699999999999995,"15min":0.746},"min":{"1min":0.44800000000000001,"5min":0.433,"15min":0.433},"max":{"1min":1.1559999999999999,"5min":1.2549999999999999,"15min":1.2549999999999999},"last":1.0409999999999999}]},{"osd":4,"last update":"Mon Jul 15 06:49:10 2024","interfaces":[{"interface":"back","average":{"1min":0.98999999999999999,"5min":0.94599999999999995,"15min":0.89400000000000002},"min":{"1min":0.60899999999999999,"5min":0.56299999999999994,"15min":0.56299999999999994},"max":{"1min":1.3120000000000001,"5min":1.3120000000000001,"15min":1.3120000000000001},"last":1.139},{"interface":"front","average":{"1min":1.0289999999999999,"5min":1.0089999999999999,"15min":0.94899999999999995},"min":{"1min":0.67900000000000005,"5min":0.50800000000000001,"15min":0.50800000000000001},"max":{"1min":1.3740000000000001,"5min":1.47,"15min":1.47},"last":1.2589999999999999}]},{"osd":5,"last update":"Mon Jul 15 06:49:10 2024","interfaces":[{"interface":"back","average":{"1min":1.042,"5min":0.997,"15min":0.96899999999999997},"min":{"1min":0.78000000000000003,"5min":0.48299999999999998,"15min":0.48299999999999998},"max":{"1min":1.415,"5min":1.571,"15min":1.571},"last":1.1739999999999999},{"interface":"front","average":{"1min":0.96399999999999997,"5min":0.97199999999999998,"15min":0.96099999999999997},"min":{"1min":0.71099999999999997,"5min":0.57699999999999996,"15min":0.57699999999999996},"max":{"1min":1.202,"5min":1.571,"15min":1.571},"last":0.99099999999999999}]},{"osd":6,"last update":"Mon Jul 15 06:49:02 2024","interfaces":[{"interface":"back","average":{"1min":1.0429999999999999,"5min":1.052,"15min":1.0649999999999999},"min":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"max":{"1min":1.339,"5min":1.548,"15min":1.548},"last":1.107},{"interface":"front","average":{"1min":0.97999999999999998,"5min":0.94799999999999995,"15min":0.95299999999999996},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":2.3500000000000001,"5min":2.3500000000000001,"15min":2.3500000000000001},"last":0.88100000000000001}]},{"osd":7,"last update":"Mon Jul 15 06:49:51 2024","interfaces":[{"interface":"back","average":{"1min":0.90800000000000003,"5min":1.036,"15min":1.0580000000000001},"min":{"1min":0.58199999999999996,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":1.3100000000000001,"5min":1.6299999999999999,"15min":1.6299999999999999},"last":1.0629999999999999},{"interface":"front","average":{"1min":1.0780000000000001,"5min":1.081,"15min":1.073},"min":{"1min":0.69599999999999995,"5min":0.50600000000000001,"15min":0.50600000000000001},"max":{"1min":1.756,"5min":1.756,"15min":1.756},"last":1.22}]}]},{"osd":1,"up_from":13,"seq":55834574952,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297972,"kb_used_data":616,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447180,"statfs":{"total":95995035648,"available":95689912320,"internally_reserved":0,"allocated":630784,"data_stored":263223,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Jul 15 06:49:45 2024","interfaces":[{"interface":"back","average":{"1min":0.64300000000000002,"5min":0.67300000000000004,"15min":0.57399999999999995},"min":{"1min":0.36199999999999999,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":1.02,"5min":1.2250000000000001,"15min":1.2250000000000001},"last":0.47499999999999998},{"interface":"front","average":{"1min":0.73699999999999999,"5min":0.72599999999999998,"15min":0.60899999999999999},"min":{"1min":0.51600000000000001,"5min":0.32700000000000001,"15min":0.312},"max":{"1min":1.181,"5min":1.5609999999999999,"15min":1.5609999999999999},"last":0.55900000000000005}]},{"osd":2,"last update":"Mon Jul 15 06:49:30 2024","interfaces":[{"interface":"back","average":{"1min":0.81499999999999995,"5min":0.76900000000000002,"15min":0.67200000000000004},"min":{"1min":0.435,"5min":0.38600000000000001,"15min":0.28299999999999997},"max":{"1min":1.0720000000000001,"5min":1.4379999999999999,"15min":1.7509999999999999},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.751,"5min":0.749,"15min":0.627},"min":{"1min":0.47699999999999998,"5min":0.30499999999999999,"15min":0.29599999999999999},"max":{"1min":1.137,"5min":1.47,"15min":1.47},"last":0.43099999999999999}]},{"osd":3,"last update":"Mon Jul 15 06:49:30 2024","interfaces":[{"interface":"back","average":{"1min":0.83099999999999996,"5min":0.80700000000000005,"15min":0.78100000000000003},"min":{"1min":0.47499999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":1.0549999999999999,"5min":1.3220000000000001,"15min":1.3220000000000001},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.82199999999999995,"5min":0.75700000000000001,"15min":0.71999999999999997},"min":{"1min":0.53500000000000003,"5min":0.377,"15min":0.377},"max":{"1min":1.431,"5min":1.431,"15min":1.431},"last":0.53900000000000003}]},{"osd":4,"last update":"Mon Jul 15 06:49:12 2024","interfaces":[{"interface":"back","average":{"1min":1.0960000000000001,"5min":0.999,"15min":0.94499999999999995},"min":{"1min":0.73599999999999999,"5min":0.66200000000000003,"15min":0.66200000000000003},"max":{"1min":1.512,"5min":1.5840000000000001,"15min":1.5840000000000001},"last":0.88200000000000001},{"interface":"front","average":{"1min":0.998,"5min":0.94999999999999996,"15min":0.92200000000000004},"min":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"max":{"1min":1.3149999999999999,"5min":1.6839999999999999,"15min":1.6839999999999999},"last":0.90400000000000003}]},{"osd":5,"last update":"Mon Jul 15 06:49:06 2024","interfaces":[{"interface":"back","average":{"1min":1.002,"5min":1.0309999999999999,"15min":1.0589999999999999},"min":{"1min":0.65900000000000003,"5min":0.63,"15min":0.63},"max":{"1min":1.5109999999999999,"5min":4.431,"15min":4.431},"last":0.68200000000000005},{"interface":"front","average":{"1min":0.996,"5min":1.024,"15min":1.036},"min":{"1min":0.76900000000000002,"5min":0.75800000000000001,"15min":0.75800000000000001},"max":{"1min":1.363,"5min":3.0670000000000002,"15min":3.0670000000000002},"last":0.82499999999999996}]},{"osd":6,"last update":"Mon Jul 15 06:49:59 2024","interfaces":[{"interface":"back","average":{"1min":1.006,"5min":1.0580000000000001,"15min":1.0700000000000001},"min":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"max":{"1min":1.405,"5min":2.0339999999999998,"15min":2.0339999999999998},"last":0.70799999999999996},{"interface":"front","average":{"1min":1.0229999999999999,"5min":1.0649999999999999,"15min":1.0900000000000001},"min":{"1min":0.78200000000000003,"5min":0.65200000000000002,"15min":0.65200000000000002},"max":{"1min":1.3560000000000001,"5min":1.897,"15min":1.897},"last":0.85999999999999999}]},{"osd":7,"last update":"Mon Jul 15 06:49:53 2024","interfaces":[{"interface":"back","average":{"1min":1.143,"5min":1.1279999999999999,"15min":1.141},"min":{"1min":0.84299999999999997,"5min":0.72299999999999998,"15min":0.72299999999999998},"max":{"1min":1.4630000000000001,"5min":1.988,"15min":1.988},"last":0.91900000000000004},{"interface":"front","average":{"1min":1.044,"5min":1.054,"15min":1.0660000000000001},"min":{"1min":0.77800000000000002,"5min":0.64200000000000002,"15min":0.64200000000000002},"max":{"1min":1.498,"5min":1.7809999999999999,"15min":1.7809999999999999},"last":0.77100000000000002}]}]},{"osd":0,"up_from":8,"seq":34359738483,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298552,"kb_used_data":1196,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446600,"statfs":{"total":95995035648,"available":95689318400,"internally_reserved":0,"allocated":1224704,"data_stored":853591,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Jul 15 06:49:39 2024","interfaces":[{"interface":"back","average":{"1min":0.66400000000000003,"5min":0.67100000000000004,"15min":0.57099999999999995},"min":{"1min":0.53200000000000003,"5min":0.33900000000000002,"15min":0.26900000000000002},"max":{"1min":0.98199999999999998,"5min":1.2070000000000001,"15min":1.2070000000000001},"last":0.58599999999999997},{"interface":"front","average":{"1min":0.79200000000000004,"5min":0.73999999999999999,"15min":0.627},"min":{"1min":0.54500000000000004,"5min":0.39200000000000002,"15min":0.28499999999999998},"max":{"1min":1.0569999999999999,"5min":1.1180000000000001,"15min":1.2110000000000001},"last":0.67400000000000004}]},{"osd":2,"last update":"Mon Jul 15 06:49:31 2024","interfaces":[{"interface":"back","average":{"1min":0.80700000000000005,"5min":0.751,"15min":0.67900000000000005},"min":{"1min":0.55000000000000004,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":1.1040000000000001,"5min":1.1040000000000001,"15min":1.1040000000000001},"last":0.84999999999999998},{"interface":"front","average":{"1min":0.72099999999999997,"5min":0.71799999999999997,"15min":0.67200000000000004},"min":{"1min":0.44900000000000001,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":1.079,"5min":1.54,"15min":1.54},"last":0.63600000000000001}]},{"osd":3,"last update":"Mon Jul 15 06:49:20 2024","interfaces":[{"interface":"back","average":{"1min":0.82999999999999996,"5min":0.80000000000000004,"15min":0.76900000000000002},"min":{"1min":0.56399999999999995,"5min":0.371,"15min":0.371},"max":{"1min":1.1679999999999999,"5min":1.7470000000000001,"15min":1.7470000000000001},"last":0.73099999999999998},{"interface":"front","average":{"1min":0.79200000000000004,"5min":0.74299999999999999,"15min":0.71599999999999997},"min":{"1min":0.65200000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.97799999999999998,"5min":0.99399999999999999,"15min":0.99399999999999999},"last":0.76400000000000001}]},{"osd":4,"last update":"Mon Jul 15 06:49:16 2024","interfaces":[{"interface":"back","average":{"1min":0.99399999999999999,"5min":0.92500000000000004,"15min":0.89700000000000002},"min":{"1min":0.64000000000000001,"5min":0.51800000000000002,"15min":0.51800000000000002},"max":{"1min":1.272,"5min":1.2909999999999999,"15min":1.2909999999999999},"last":1.028},{"interface":"front","average":{"1min":1.0820000000000001,"5min":0.99099999999999999,"15min":0.93100000000000005},"min":{"1min":0.82199999999999995,"5min":0.52800000000000002,"15min":0.52800000000000002},"max":{"1min":1.234,"5min":1.498,"15min":1.498},"last":1.119}]},{"osd":5,"last update":"Mon Jul 15 06:49:04 2024","interfaces":[{"interface":"back","average":{"1min":0.91700000000000004,"5min":0.92400000000000004,"15min":0.94699999999999995},"min":{"1min":0.73599999999999999,"5min":0.58399999999999996,"15min":0.58399999999999996},"max":{"1min":1.23,"5min":1.3280000000000001,"15min":1.3280000000000001},"last":1.0629999999999999},{"interface":"front","average":{"1min":1.0129999999999999,"5min":0.95899999999999996,"15min":0.95599999999999996},"min":{"1min":0.73799999999999999,"5min":0.51000000000000001,"15min":0.51000000000000001},"max":{"1min":1.3160000000000001,"5min":1.4350000000000001,"15min":1.4350000000000001},"last":0.89400000000000002}]},{"osd":6,"last update":"Mon Jul 15 06:49:59 2024","interfaces":[{"interface":"back","average":{"1min":1.0509999999999999,"5min":0.998,"15min":0.98199999999999998},"min":{"1min":0.55000000000000004,"5min":0.5,"15min":0.5},"max":{"1min":1.359,"5min":1.359,"15min":1.359},"last":1.0900000000000001},{"interface":"front","average":{"1min":0.997,"5min":1.03,"15min":1.0429999999999999},"min":{"1min":0.77200000000000002,"5min":0.47799999999999998,"15min":0.47799999999999998},"max":{"1min":1.2869999999999999,"5min":1.4039999999999999,"15min":1.4039999999999999},"last":0.95299999999999996}]},{"osd":7,"last update":"Mon Jul 15 06:49:50 2024","interfaces":[{"interface":"back","average":{"1min":1.091,"5min":1.0609999999999999,"15min":1.052},"min":{"1min":0.68500000000000005,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":1.3089999999999999,"5min":1.3200000000000001,"15min":1.3200000000000001},"last":0.93500000000000005},{"interface":"front","average":{"1min":1.054,"5min":1.069,"15min":1.0549999999999999},"min":{"1min":0.93300000000000005,"5min":0.60999999999999999,"15min":0.60999999999999999},"max":{"1min":1.3640000000000001,"5min":1.827,"15min":1.827},"last":0.995}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-07-15T06:50:01.800 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph pg dump --format=json 2024-07-15T06:50:02.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:02 smithi134 bash[24068]: audit 2024-07-15T06:50:01.122226+0000 mgr.y (mgr.24272) 102 : audit [DBG] from='client.24455 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-07-15T06:50:02.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:02 smithi089 bash[21793]: audit 2024-07-15T06:50:01.122226+0000 mgr.y (mgr.24272) 102 : audit [DBG] from='client.24455 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-07-15T06:50:02.619 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:02 smithi089 bash[31372]: audit 2024-07-15T06:50:01.122226+0000 mgr.y (mgr.24272) 102 : audit [DBG] from='client.24455 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-07-15T06:50:03.119 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:50:02 smithi089 bash[22045]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:50:02] "GET /metrics HTTP/1.1" 200 203246 "" "Prometheus/2.43.0" 2024-07-15T06:50:03.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:03 smithi134 bash[24068]: cluster 2024-07-15T06:50:02.143029+0000 mgr.y (mgr.24272) 103 : cluster [DBG] pgmap v76: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:50:03.619 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:03 smithi089 bash[31372]: cluster 2024-07-15T06:50:02.143029+0000 mgr.y (mgr.24272) 103 : cluster [DBG] pgmap v76: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:50:03.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:03 smithi089 bash[21793]: cluster 2024-07-15T06:50:02.143029+0000 mgr.y (mgr.24272) 103 : cluster [DBG] pgmap v76: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:50:05.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:05 smithi134 bash[24068]: cluster 2024-07-15T06:50:04.143624+0000 mgr.y (mgr.24272) 104 : cluster [DBG] pgmap v77: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:50:05.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:05 smithi134 bash[24068]: audit 2024-07-15T06:50:05.259689+0000 mon.a (mon.0) 789 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:50:05.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:05 smithi089 bash[21793]: cluster 2024-07-15T06:50:04.143624+0000 mgr.y (mgr.24272) 104 : cluster [DBG] pgmap v77: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:50:05.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:05 smithi089 bash[21793]: audit 2024-07-15T06:50:05.259689+0000 mon.a (mon.0) 789 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:50:05.619 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:05 smithi089 bash[31372]: cluster 2024-07-15T06:50:04.143624+0000 mgr.y (mgr.24272) 104 : cluster [DBG] pgmap v77: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:50:05.619 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:05 smithi089 bash[31372]: audit 2024-07-15T06:50:05.259689+0000 mon.a (mon.0) 789 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:50:06.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:06 smithi134 bash[24068]: audit 2024-07-15T06:50:05.748334+0000 mon.a (mon.0) 790 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:50:06.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:06 smithi134 bash[24068]: audit 2024-07-15T06:50:05.750102+0000 mon.a (mon.0) 791 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:50:06.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:06 smithi134 bash[24068]: audit 2024-07-15T06:50:05.759655+0000 mon.a (mon.0) 792 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:50:06.576 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T06:50:06.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:06 smithi089 bash[21793]: audit 2024-07-15T06:50:05.748334+0000 mon.a (mon.0) 790 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:50:06.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:06 smithi089 bash[21793]: audit 2024-07-15T06:50:05.750102+0000 mon.a (mon.0) 791 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:50:06.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:06 smithi089 bash[21793]: audit 2024-07-15T06:50:05.759655+0000 mon.a (mon.0) 792 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:50:06.619 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:06 smithi089 bash[31372]: audit 2024-07-15T06:50:05.748334+0000 mon.a (mon.0) 790 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:50:06.619 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:06 smithi089 bash[31372]: audit 2024-07-15T06:50:05.750102+0000 mon.a (mon.0) 791 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:50:06.619 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:06 smithi089 bash[31372]: audit 2024-07-15T06:50:05.759655+0000 mon.a (mon.0) 792 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:50:07.304 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:50:06 smithi134 bash[25019]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:50:06] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-07-15T06:50:07.970 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:07 smithi089 bash[31372]: cluster 2024-07-15T06:50:06.144300+0000 mgr.y (mgr.24272) 105 : cluster [DBG] pgmap v78: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:50:07.970 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:07 smithi089 bash[21793]: cluster 2024-07-15T06:50:06.144300+0000 mgr.y (mgr.24272) 105 : cluster [DBG] pgmap v78: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:50:08.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:07 smithi134 bash[24068]: cluster 2024-07-15T06:50:06.144300+0000 mgr.y (mgr.24272) 105 : cluster [DBG] pgmap v78: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:50:09.852 INFO:teuthology.orchestra.run.smithi089.stderr:dumped all 2024-07-15T06:50:09.853 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-07-15T06:50:10.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:09 smithi134 bash[24068]: cluster 2024-07-15T06:50:08.144859+0000 mgr.y (mgr.24272) 106 : cluster [DBG] pgmap v79: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:50:10.078 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:09 smithi089 bash[31372]: cluster 2024-07-15T06:50:08.144859+0000 mgr.y (mgr.24272) 106 : cluster [DBG] pgmap v79: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:50:10.078 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:09 smithi089 bash[21793]: cluster 2024-07-15T06:50:08.144859+0000 mgr.y (mgr.24272) 106 : cluster [DBG] pgmap v79: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:50:10.503 INFO:teuthology.orchestra.run.smithi089.stdout:{"pg_ready":true,"pg_map":{"version":79,"stamp":"2024-07-15T06:50:08.144599+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2385508,"kb_used_data":6660,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747575708,"statfs":{"total":767960285184,"available":765517524992,"internally_reserved":0,"allocated":6819840,"data_stored":3876242,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003582"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2024-07-15T06:48:29.972552+0000","last_change":"2024-07-15T06:48:28.675814+0000","last_active":"2024-07-15T06:48:29.972552+0000","last_peered":"2024-07-15T06:48:29.972552+0000","last_clean":"2024-07-15T06:48:29.972552+0000","last_became_active":"2024-07-15T06:48:28.674798+0000","last_became_peered":"2024-07-15T06:48:28.674798+0000","last_unstale":"2024-07-15T06:48:29.972552+0000","last_undegraded":"2024-07-15T06:48:29.972552+0000","last_fullsized":"2024-07-15T06:48:29.972552+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_clean_scrub_stamp":"2024-07-15T06:48:27.655327+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T14:36:19.382840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2024-07-15T06:48:30.120901+0000","last_change":"2024-07-15T06:48:28.872731+0000","last_active":"2024-07-15T06:48:30.120901+0000","last_peered":"2024-07-15T06:48:30.120901+0000","last_clean":"2024-07-15T06:48:30.120901+0000","last_became_active":"2024-07-15T06:48:28.871744+0000","last_became_peered":"2024-07-15T06:48:28.871744+0000","last_unstale":"2024-07-15T06:48:30.120901+0000","last_undegraded":"2024-07-15T06:48:30.120901+0000","last_fullsized":"2024-07-15T06:48:30.120901+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_clean_scrub_stamp":"2024-07-15T06:48:27.655327+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T11:36:09.487214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2024-07-15T06:48:29.669133+0000","last_change":"2024-07-15T06:48:28.872322+0000","last_active":"2024-07-15T06:48:29.669133+0000","last_peered":"2024-07-15T06:48:29.669133+0000","last_clean":"2024-07-15T06:48:29.669133+0000","last_became_active":"2024-07-15T06:48:28.871320+0000","last_became_peered":"2024-07-15T06:48:28.871320+0000","last_unstale":"2024-07-15T06:48:29.669133+0000","last_undegraded":"2024-07-15T06:48:29.669133+0000","last_fullsized":"2024-07-15T06:48:29.669133+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_clean_scrub_stamp":"2024-07-15T06:48:27.655327+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T14:24:12.762681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2024-07-15T06:48:30.120812+0000","last_change":"2024-07-15T06:48:28.738368+0000","last_active":"2024-07-15T06:48:30.120812+0000","last_peered":"2024-07-15T06:48:30.120812+0000","last_clean":"2024-07-15T06:48:30.120812+0000","last_became_active":"2024-07-15T06:48:28.737382+0000","last_became_peered":"2024-07-15T06:48:28.737382+0000","last_unstale":"2024-07-15T06:48:30.120812+0000","last_undegraded":"2024-07-15T06:48:30.120812+0000","last_fullsized":"2024-07-15T06:48:30.120812+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_clean_scrub_stamp":"2024-07-15T06:48:27.655327+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T17:01:42.558382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2024-07-15T06:48:30.405779+0000","last_change":"2024-07-15T06:48:28.674943+0000","last_active":"2024-07-15T06:48:30.405779+0000","last_peered":"2024-07-15T06:48:30.405779+0000","last_clean":"2024-07-15T06:48:30.405779+0000","last_became_active":"2024-07-15T06:48:28.674174+0000","last_became_peered":"2024-07-15T06:48:28.674174+0000","last_unstale":"2024-07-15T06:48:30.405779+0000","last_undegraded":"2024-07-15T06:48:30.405779+0000","last_fullsized":"2024-07-15T06:48:30.405779+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_clean_scrub_stamp":"2024-07-15T06:48:27.655327+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T17:40:56.394801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2024-07-15T06:48:29.665606+0000","last_change":"2024-07-15T06:48:28.738441+0000","last_active":"2024-07-15T06:48:29.665606+0000","last_peered":"2024-07-15T06:48:29.665606+0000","last_clean":"2024-07-15T06:48:29.665606+0000","last_became_active":"2024-07-15T06:48:28.737053+0000","last_became_peered":"2024-07-15T06:48:28.737053+0000","last_unstale":"2024-07-15T06:48:29.665606+0000","last_undegraded":"2024-07-15T06:48:29.665606+0000","last_fullsized":"2024-07-15T06:48:29.665606+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_clean_scrub_stamp":"2024-07-15T06:48:27.655327+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T08:33:33.092342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2024-07-15T06:48:29.669009+0000","last_change":"2024-07-15T06:48:28.737091+0000","last_active":"2024-07-15T06:48:29.669009+0000","last_peered":"2024-07-15T06:48:29.669009+0000","last_clean":"2024-07-15T06:48:29.669009+0000","last_became_active":"2024-07-15T06:48:28.736574+0000","last_became_peered":"2024-07-15T06:48:28.736574+0000","last_unstale":"2024-07-15T06:48:29.669009+0000","last_undegraded":"2024-07-15T06:48:29.669009+0000","last_fullsized":"2024-07-15T06:48:29.669009+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_clean_scrub_stamp":"2024-07-15T06:48:27.655327+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T09:31:07.195677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2024-07-15T06:48:30.405734+0000","last_change":"2024-07-15T06:48:28.672045+0000","last_active":"2024-07-15T06:48:30.405734+0000","last_peered":"2024-07-15T06:48:30.405734+0000","last_clean":"2024-07-15T06:48:30.405734+0000","last_became_active":"2024-07-15T06:48:28.671290+0000","last_became_peered":"2024-07-15T06:48:28.671290+0000","last_unstale":"2024-07-15T06:48:30.405734+0000","last_undegraded":"2024-07-15T06:48:30.405734+0000","last_fullsized":"2024-07-15T06:48:30.405734+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_clean_scrub_stamp":"2024-07-15T06:48:27.655327+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T11:10:59.552676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"59'128","reported_seq":270,"reported_epoch":62,"state":"active+clean","last_fresh":"2024-07-15T06:48:29.669027+0000","last_change":"2024-07-15T06:46:49.356522+0000","last_active":"2024-07-15T06:48:29.669027+0000","last_peered":"2024-07-15T06:48:29.669027+0000","last_clean":"2024-07-15T06:48:29.669027+0000","last_became_active":"2024-07-15T06:46:49.355501+0000","last_became_peered":"2024-07-15T06:46:49.355501+0000","last_unstale":"2024-07-15T06:48:29.669027+0000","last_undegraded":"2024-07-15T06:48:29.669027+0000","last_fullsized":"2024-07-15T06:48:29.669027+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:42:21.582843+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:42:21.582843+0000","last_clean_scrub_stamp":"2024-07-15T06:42:21.582843+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T10:17:36.460676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234027,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298548,"kb_used_data":1192,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446604,"statfs":{"total":95995035648,"available":95689322496,"internally_reserved":0,"allocated":1220608,"data_stored":853268,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Jul 15 06:49:52 2024","interfaces":[{"interface":"back","average":{"1min":0.89100000000000001,"5min":0.92400000000000004,"15min":0.93300000000000005},"min":{"1min":0.49099999999999999,"5min":0.443,"15min":0.443},"max":{"1min":1.532,"5min":1.532,"15min":1.532},"last":0.79400000000000004},{"interface":"front","average":{"1min":0.86199999999999999,"5min":0.879,"15min":0.89300000000000002},"min":{"1min":0.52300000000000002,"5min":0.46600000000000003,"15min":0.46600000000000003},"max":{"1min":1.2609999999999999,"5min":1.2609999999999999,"15min":1.2609999999999999},"last":1.0620000000000001}]},{"osd":1,"last update":"Mon Jul 15 06:49:52 2024","interfaces":[{"interface":"back","average":{"1min":0.84199999999999997,"5min":0.92000000000000004,"15min":0.93500000000000005},"min":{"1min":0.52200000000000002,"5min":0.48499999999999999,"15min":0.48499999999999999},"max":{"1min":1.6579999999999999,"5min":1.6579999999999999,"15min":1.6579999999999999},"last":0.83799999999999997},{"interface":"front","average":{"1min":0.91100000000000003,"5min":0.96899999999999997,"15min":0.999},"min":{"1min":0.51500000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"max":{"1min":1.3069999999999999,"5min":1.357,"15min":1.357},"last":0.94799999999999995}]},{"osd":2,"last update":"Mon Jul 15 06:49:52 2024","interfaces":[{"interface":"back","average":{"1min":0.92600000000000005,"5min":0.92900000000000005,"15min":0.91200000000000003},"min":{"1min":0.67800000000000005,"5min":0.50900000000000001,"15min":0.50900000000000001},"max":{"1min":1.6279999999999999,"5min":1.6279999999999999,"15min":1.6279999999999999},"last":0.90600000000000003},{"interface":"front","average":{"1min":0.91700000000000004,"5min":0.96799999999999997,"15min":0.97999999999999998},"min":{"1min":0.59299999999999997,"5min":0.55800000000000005,"15min":0.55800000000000005},"max":{"1min":1.802,"5min":1.802,"15min":1.802},"last":0.88700000000000001}]},{"osd":3,"last update":"Mon Jul 15 06:49:52 2024","interfaces":[{"interface":"back","average":{"1min":0.92100000000000004,"5min":0.95099999999999996,"15min":0.94599999999999995},"min":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"max":{"1min":1.704,"5min":1.704,"15min":1.704},"last":0.97799999999999998},{"interface":"front","average":{"1min":1.0089999999999999,"5min":1.0409999999999999,"15min":1.0620000000000001},"min":{"1min":0.75800000000000001,"5min":0.60499999999999998,"15min":0.60499999999999998},"max":{"1min":1.74,"5min":1.74,"15min":1.74},"last":0.999}]},{"osd":4,"last update":"Mon Jul 15 06:49:52 2024","interfaces":[{"interface":"back","average":{"1min":0.97499999999999998,"5min":0.96399999999999997,"15min":0.97099999999999997},"min":{"1min":0.59699999999999998,"5min":0.52500000000000002,"15min":0.52500000000000002},"max":{"1min":1.2889999999999999,"5min":1.456,"15min":1.456},"last":0.71199999999999997},{"interface":"front","average":{"1min":1.016,"5min":1.0389999999999999,"15min":1.0620000000000001},"min":{"1min":0.64000000000000001,"5min":0.57099999999999995,"15min":0.57099999999999995},"max":{"1min":2.0630000000000002,"5min":2.0630000000000002,"15min":2.0630000000000002},"last":1.1100000000000001}]},{"osd":5,"last update":"Mon Jul 15 06:49:52 2024","interfaces":[{"interface":"back","average":{"1min":0.97899999999999998,"5min":1.0189999999999999,"15min":1.0469999999999999},"min":{"1min":0.63400000000000001,"5min":0.499,"15min":0.499},"max":{"1min":1.8680000000000001,"5min":1.8680000000000001,"15min":1.8680000000000001},"last":0.74399999999999999},{"interface":"front","average":{"1min":1.073,"5min":1.056,"15min":1.077},"min":{"1min":0.78500000000000003,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":1.7729999999999999,"5min":1.7729999999999999,"15min":1.7729999999999999},"last":1.22}]},{"osd":6,"last update":"Mon Jul 15 06:49:52 2024","interfaces":[{"interface":"back","average":{"1min":1.125,"5min":1.1000000000000001,"15min":1.1240000000000001},"min":{"1min":0.80500000000000005,"5min":0.50600000000000001,"15min":0.50600000000000001},"max":{"1min":1.9079999999999999,"5min":1.9079999999999999,"15min":1.9079999999999999},"last":0.85799999999999998},{"interface":"front","average":{"1min":1.107,"5min":1.111,"15min":1.131},"min":{"1min":0.71599999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"max":{"1min":1.823,"5min":1.823,"15min":1.823},"last":1.028}]}]},{"osd":6,"up_from":46,"seq":197568495669,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298552,"kb_used_data":1196,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446600,"statfs":{"total":95995035648,"available":95689318400,"internally_reserved":0,"allocated":1224704,"data_stored":853591,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Jul 15 06:49:57 2024","interfaces":[{"interface":"back","average":{"1min":0.91600000000000004,"5min":0.88800000000000001,"15min":0.91800000000000004},"min":{"1min":0.52500000000000002,"5min":0.50700000000000001,"15min":0.50700000000000001},"max":{"1min":1.181,"5min":2.1579999999999999,"15min":2.1579999999999999},"last":0.70799999999999996},{"interface":"front","average":{"1min":0.91600000000000004,"5min":0.88200000000000001,"15min":0.89200000000000002},"min":{"1min":0.627,"5min":0.47099999999999997,"15min":0.47099999999999997},"max":{"1min":1.276,"5min":1.3089999999999999,"15min":1.3089999999999999},"last":0.84699999999999998}]},{"osd":1,"last update":"Mon Jul 15 06:49:57 2024","interfaces":[{"interface":"back","average":{"1min":0.97599999999999998,"5min":0.91700000000000004,"15min":0.91500000000000004},"min":{"1min":0.71999999999999997,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":1.4019999999999999,"5min":1.444,"15min":1.444},"last":0.81599999999999995},{"interface":"front","average":{"1min":0.89000000000000001,"5min":0.91100000000000003,"15min":0.95699999999999996},"min":{"1min":0.60999999999999999,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":1.4119999999999999,"5min":2.2210000000000001,"15min":2.2210000000000001},"last":0.622}]},{"osd":2,"last update":"Mon Jul 15 06:49:57 2024","interfaces":[{"interface":"back","average":{"1min":0.84299999999999997,"5min":0.91400000000000003,"15min":0.97699999999999998},"min":{"1min":0.67700000000000005,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":1.139,"5min":2.754,"15min":2.754},"last":0.88800000000000001},{"interface":"front","average":{"1min":1,"5min":0.96199999999999997,"15min":1.004},"min":{"1min":0.73999999999999999,"5min":0.501,"15min":0.501},"max":{"1min":1.266,"5min":2.1019999999999999,"15min":2.1019999999999999},"last":0.77300000000000002}]},{"osd":3,"last update":"Mon Jul 15 06:49:57 2024","interfaces":[{"interface":"back","average":{"1min":0.97399999999999998,"5min":0.91200000000000003,"15min":0.90600000000000003},"min":{"1min":0.73399999999999999,"5min":0.498,"15min":0.498},"max":{"1min":1.224,"5min":1.3799999999999999,"15min":1.3799999999999999},"last":1.069},{"interface":"front","average":{"1min":0.999,"5min":0.95299999999999996,"15min":0.997},"min":{"1min":0.79600000000000004,"5min":0.52500000000000002,"15min":0.52500000000000002},"max":{"1min":1.321,"5min":2.0600000000000001,"15min":2.0600000000000001},"last":1.0369999999999999}]},{"osd":4,"last update":"Mon Jul 15 06:49:57 2024","interfaces":[{"interface":"back","average":{"1min":0.90300000000000002,"5min":0.79700000000000004,"15min":0.78700000000000003},"min":{"1min":0.624,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":1.165,"5min":1.6120000000000001,"15min":1.6120000000000001},"last":1.1819999999999999},{"interface":"front","average":{"1min":1.004,"5min":0.88500000000000001,"15min":0.85699999999999998},"min":{"1min":0.67500000000000004,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":1.417,"5min":1.417,"15min":1.417},"last":1.141}]},{"osd":5,"last update":"Mon Jul 15 06:49:57 2024","interfaces":[{"interface":"back","average":{"1min":1.0329999999999999,"5min":0.89100000000000001,"15min":0.86899999999999999},"min":{"1min":0.72999999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":1.256,"5min":1.6779999999999999,"15min":1.6779999999999999},"last":0.98799999999999999},{"interface":"front","average":{"1min":1.1240000000000001,"5min":0.92000000000000004,"15min":0.873},"min":{"1min":0.78300000000000003,"5min":0.373,"15min":0.373},"max":{"1min":1.3520000000000001,"5min":1.375,"15min":1.375},"last":0.73499999999999999}]},{"osd":7,"last update":"Mon Jul 15 06:49:48 2024","interfaces":[{"interface":"back","average":{"1min":1.097,"5min":0.998,"15min":0.98899999999999999},"min":{"1min":0.53400000000000003,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":1.373,"5min":1.7430000000000001,"15min":1.7430000000000001},"last":1.103},{"interface":"front","average":{"1min":1.0609999999999999,"5min":1.004,"15min":0.995},"min":{"1min":0.624,"5min":0.435,"15min":0.435},"max":{"1min":1.4530000000000001,"5min":1.8540000000000001,"15min":1.8540000000000001},"last":0.94799999999999995}]}]},{"osd":5,"up_from":38,"seq":163208757311,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297968,"kb_used_data":612,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447184,"statfs":{"total":95995035648,"available":95689916416,"internally_reserved":0,"allocated":626688,"data_stored":262900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Jul 15 06:49:06 2024","interfaces":[{"interface":"back","average":{"1min":0.88300000000000001,"5min":0.92600000000000005,"15min":0.92200000000000004},"min":{"1min":0.48199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"max":{"1min":1.135,"5min":1.1859999999999999,"15min":1.1859999999999999},"last":1.226},{"interface":"front","average":{"1min":0.91700000000000004,"5min":0.89900000000000002,"15min":0.88900000000000001},"min":{"1min":0.55600000000000005,"5min":0.502,"15min":0.502},"max":{"1min":1.177,"5min":1.2210000000000001,"15min":1.2210000000000001},"last":0.877}]},{"osd":1,"last update":"Mon Jul 15 06:49:06 2024","interfaces":[{"interface":"back","average":{"1min":0.86199999999999999,"5min":0.83699999999999997,"15min":0.81100000000000005},"min":{"1min":0.54500000000000004,"5min":0.53800000000000003,"15min":0.53800000000000003},"max":{"1min":1.1850000000000001,"5min":1.1850000000000001,"15min":1.1850000000000001},"last":1.1399999999999999},{"interface":"front","average":{"1min":0.86299999999999999,"5min":0.873,"15min":0.88300000000000001},"min":{"1min":0.52500000000000002,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":1.208,"5min":1.268,"15min":1.268},"last":1.0349999999999999}]},{"osd":2,"last update":"Mon Jul 15 06:49:06 2024","interfaces":[{"interface":"back","average":{"1min":0.89700000000000002,"5min":0.84199999999999997,"15min":0.81699999999999995},"min":{"1min":0.54600000000000004,"5min":0.52500000000000002,"15min":0.52500000000000002},"max":{"1min":1.2869999999999999,"5min":1.2869999999999999,"15min":1.2869999999999999},"last":0.85399999999999998},{"interface":"front","average":{"1min":0.86599999999999999,"5min":0.89700000000000002,"15min":0.91300000000000003},"min":{"1min":0.68799999999999994,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":1.1060000000000001,"5min":1.286,"15min":1.286},"last":1.0629999999999999}]},{"osd":3,"last update":"Mon Jul 15 06:49:06 2024","interfaces":[{"interface":"back","average":{"1min":0.90300000000000002,"5min":0.85999999999999999,"15min":0.84599999999999997},"min":{"1min":0.61399999999999999,"5min":0.56399999999999995,"15min":0.56399999999999995},"max":{"1min":1.2709999999999999,"5min":1.2709999999999999,"15min":1.2709999999999999},"last":0.77900000000000003},{"interface":"front","average":{"1min":0.872,"5min":0.93400000000000005,"15min":0.94199999999999995},"min":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"max":{"1min":1.23,"5min":1.3320000000000001,"15min":1.3320000000000001},"last":0.92400000000000004}]},{"osd":4,"last update":"Mon Jul 15 06:49:06 2024","interfaces":[{"interface":"back","average":{"1min":0.79900000000000004,"5min":0.748,"15min":0.72599999999999998},"min":{"1min":0.36799999999999999,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":1.091,"5min":1.1910000000000001,"15min":1.1910000000000001},"last":0.623},{"interface":"front","average":{"1min":0.85199999999999998,"5min":0.79500000000000004,"15min":0.75900000000000001},"min":{"1min":0.41999999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":1.3420000000000001,"5min":1.3420000000000001,"15min":1.3420000000000001},"last":0.66800000000000004}]},{"osd":6,"last update":"Mon Jul 15 06:49:55 2024","interfaces":[{"interface":"back","average":{"1min":0.99199999999999999,"5min":0.89500000000000002,"15min":0.89600000000000002},"min":{"1min":0.68200000000000005,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":1.2250000000000001,"5min":1.256,"15min":1.256},"last":1.1859999999999999},{"interface":"front","average":{"1min":0.97999999999999998,"5min":0.91900000000000004,"15min":0.91600000000000004},"min":{"1min":0.69399999999999995,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":1.216,"5min":1.3049999999999999,"15min":1.3049999999999999},"last":1.1000000000000001}]},{"osd":7,"last update":"Mon Jul 15 06:49:47 2024","interfaces":[{"interface":"back","average":{"1min":1.0449999999999999,"5min":0.98899999999999999,"15min":0.97699999999999998},"min":{"1min":0.68899999999999995,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":1.256,"5min":1.3380000000000001,"15min":1.3380000000000001},"last":0.71299999999999997},{"interface":"front","average":{"1min":1.022,"5min":0.98099999999999998,"15min":0.98299999999999998},"min":{"1min":0.77500000000000002,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":1.202,"5min":1.3939999999999999,"15min":1.3939999999999999},"last":0.97099999999999997}]}]},{"osd":4,"up_from":30,"seq":128849018954,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297972,"kb_used_data":616,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447180,"statfs":{"total":95995035648,"available":95689912320,"internally_reserved":0,"allocated":630784,"data_stored":263223,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Jul 15 06:49:13 2024","interfaces":[{"interface":"back","average":{"1min":0.84399999999999997,"5min":0.84699999999999998,"15min":0.82699999999999996},"min":{"1min":0.58299999999999996,"5min":0.46600000000000003,"15min":0.46600000000000003},"max":{"1min":1.381,"5min":1.381,"15min":1.381},"last":1.212},{"interface":"front","average":{"1min":0.93100000000000005,"5min":0.88600000000000001,"15min":0.83299999999999996},"min":{"1min":0.53400000000000003,"5min":0.49099999999999999,"15min":0.49099999999999999},"max":{"1min":1.381,"5min":1.381,"15min":1.381},"last":1.0620000000000001}]},{"osd":1,"last update":"Mon Jul 15 06:49:13 2024","interfaces":[{"interface":"back","average":{"1min":0.93200000000000005,"5min":0.90300000000000002,"15min":0.86099999999999999},"min":{"1min":0.64000000000000001,"5min":0.497,"15min":0.497},"max":{"1min":1.3120000000000001,"5min":1.3120000000000001,"15min":1.3120000000000001},"last":1.256},{"interface":"front","average":{"1min":0.83299999999999996,"5min":0.84199999999999997,"15min":0.86499999999999999},"min":{"1min":0.54300000000000004,"5min":0.501,"15min":0.501},"max":{"1min":1.0960000000000001,"5min":1.3260000000000001,"15min":1.3260000000000001},"last":0.84699999999999998}]},{"osd":2,"last update":"Mon Jul 15 06:49:13 2024","interfaces":[{"interface":"back","average":{"1min":0.92600000000000005,"5min":0.873,"15min":0.83299999999999996},"min":{"1min":0.64400000000000002,"5min":0.49399999999999999,"15min":0.49399999999999999},"max":{"1min":1.204,"5min":1.204,"15min":1.204},"last":1.103},{"interface":"front","average":{"1min":0.84199999999999997,"5min":0.86399999999999999,"15min":0.872},"min":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"max":{"1min":1.9910000000000001,"5min":1.9910000000000001,"15min":1.9910000000000001},"last":1.042}]},{"osd":3,"last update":"Mon Jul 15 06:49:13 2024","interfaces":[{"interface":"back","average":{"1min":0.98499999999999999,"5min":0.89500000000000002,"15min":0.86599999999999999},"min":{"1min":0.69799999999999995,"5min":0.56000000000000005,"15min":0.56000000000000005},"max":{"1min":2.1709999999999998,"5min":2.1709999999999998,"15min":2.1709999999999998},"last":0.72699999999999998},{"interface":"front","average":{"1min":0.95899999999999996,"5min":0.91500000000000004,"15min":0.90000000000000002},"min":{"1min":0.51400000000000001,"5min":0.48999999999999999,"15min":0.48999999999999999},"max":{"1min":1.4970000000000001,"5min":1.4970000000000001,"15min":1.4970000000000001},"last":1.1439999999999999}]},{"osd":5,"last update":"Mon Jul 15 06:49:07 2024","interfaces":[{"interface":"back","average":{"1min":0.94599999999999995,"5min":0.78300000000000003,"15min":0.72799999999999998},"min":{"1min":0.40200000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":2.085,"5min":2.085,"15min":2.085},"last":0.94899999999999995},{"interface":"front","average":{"1min":0.90000000000000002,"5min":0.751,"15min":0.71499999999999997},"min":{"1min":0.32200000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":1.2729999999999999,"5min":1.2729999999999999,"15min":1.2729999999999999},"last":1.1779999999999999}]},{"osd":6,"last update":"Mon Jul 15 06:49:58 2024","interfaces":[{"interface":"back","average":{"1min":0.97999999999999998,"5min":0.85099999999999998,"15min":0.76000000000000001},"min":{"1min":0.42399999999999999,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":1.165,"5min":1.5760000000000001,"15min":1.5760000000000001},"last":0.90000000000000002},{"interface":"front","average":{"1min":1.0029999999999999,"5min":0.87,"15min":0.78800000000000003},"min":{"1min":0.47699999999999998,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":1.246,"5min":1.347,"15min":1.347},"last":1.0029999999999999}]},{"osd":7,"last update":"Mon Jul 15 06:49:48 2024","interfaces":[{"interface":"back","average":{"1min":1.0149999999999999,"5min":0.95399999999999996,"15min":0.92800000000000005},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":2.2469999999999999,"5min":2.2469999999999999,"15min":2.2469999999999999},"last":1.0840000000000001},{"interface":"front","average":{"1min":1.0369999999999999,"5min":1.0629999999999999,"15min":1.0629999999999999},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":1.653,"5min":1.653,"15min":1.653},"last":1.296}]}]},{"osd":3,"up_from":25,"seq":107374182485,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297972,"kb_used_data":616,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447180,"statfs":{"total":95995035648,"available":95689912320,"internally_reserved":0,"allocated":630784,"data_stored":263223,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Jul 15 06:49:22 2024","interfaces":[{"interface":"back","average":{"1min":0.64600000000000002,"5min":0.67100000000000004,"15min":0.64400000000000002},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.90300000000000002,"5min":1.1220000000000001,"15min":1.1220000000000001},"last":0.754},{"interface":"front","average":{"1min":0.73199999999999998,"5min":0.72399999999999998,"15min":0.69999999999999996},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":1.1060000000000001,"5min":1.647,"15min":1.647},"last":0.64600000000000002}]},{"osd":1,"last update":"Mon Jul 15 06:49:22 2024","interfaces":[{"interface":"back","average":{"1min":0.70099999999999996,"5min":0.75600000000000001,"15min":0.70799999999999996},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":1.1739999999999999,"5min":1.3580000000000001,"15min":1.3580000000000001},"last":0.73399999999999999},{"interface":"front","average":{"1min":0.75,"5min":0.73599999999999999,"15min":0.69799999999999995},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":1.0149999999999999,"5min":1.2190000000000001,"15min":1.2190000000000001},"last":0.60799999999999998}]},{"osd":2,"last update":"Mon Jul 15 06:49:22 2024","interfaces":[{"interface":"back","average":{"1min":0.77400000000000002,"5min":0.79300000000000004,"15min":0.753},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":1.157,"5min":1.157,"15min":1.157},"last":0.67900000000000005},{"interface":"front","average":{"1min":0.73899999999999999,"5min":0.76600000000000001,"15min":0.72299999999999998},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":1.3240000000000001,"5min":1.3240000000000001,"15min":1.3240000000000001},"last":0.57299999999999995}]},{"osd":4,"last update":"Mon Jul 15 06:49:17 2024","interfaces":[{"interface":"back","average":{"1min":0.92200000000000004,"5min":0.91800000000000004,"15min":0.876},"min":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"max":{"1min":1.179,"5min":1.2949999999999999,"15min":1.2949999999999999},"last":0.82799999999999996},{"interface":"front","average":{"1min":0.94999999999999996,"5min":0.94999999999999996,"15min":0.90300000000000002},"min":{"1min":0.72399999999999998,"5min":0.60899999999999999,"15min":0.60899999999999999},"max":{"1min":1.538,"5min":1.8140000000000001,"15min":1.8140000000000001},"last":0.83899999999999997}]},{"osd":5,"last update":"Mon Jul 15 06:49:11 2024","interfaces":[{"interface":"back","average":{"1min":0.97599999999999998,"5min":0.97199999999999998,"15min":0.96199999999999997},"min":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"max":{"1min":1.48,"5min":1.7609999999999999,"15min":1.7609999999999999},"last":0.873},{"interface":"front","average":{"1min":0.96799999999999997,"5min":0.98099999999999998,"15min":0.98599999999999999},"min":{"1min":0.70999999999999996,"5min":0.68100000000000005,"15min":0.68100000000000005},"max":{"1min":1.4019999999999999,"5min":1.5069999999999999,"15min":1.5069999999999999},"last":0.70299999999999996}]},{"osd":6,"last update":"Mon Jul 15 06:49:02 2024","interfaces":[{"interface":"back","average":{"1min":1.016,"5min":1.0620000000000001,"15min":1.0840000000000001},"min":{"1min":0.65600000000000003,"5min":0.64500000000000002,"15min":0.64500000000000002},"max":{"1min":1.5880000000000001,"5min":1.8899999999999999,"15min":1.8899999999999999},"last":0.93700000000000006},{"interface":"front","average":{"1min":0.97099999999999997,"5min":1.006,"15min":1.016},"min":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"max":{"1min":1.3859999999999999,"5min":1.3859999999999999,"15min":1.3859999999999999},"last":0.85999999999999999}]},{"osd":7,"last update":"Mon Jul 15 06:49:54 2024","interfaces":[{"interface":"back","average":{"1min":0.94599999999999995,"5min":1.016,"15min":1.0209999999999999},"min":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"max":{"1min":1.155,"5min":1.4730000000000001,"15min":1.4730000000000001},"last":0.76900000000000002},{"interface":"front","average":{"1min":0.98099999999999998,"5min":1.0680000000000001,"15min":1.0940000000000001},"min":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"max":{"1min":1.2589999999999999,"5min":1.6100000000000001,"15min":1.6100000000000001},"last":0.85199999999999998}]}]},{"osd":2,"up_from":18,"seq":77309411422,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297972,"kb_used_data":616,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447180,"statfs":{"total":95995035648,"available":95689912320,"internally_reserved":0,"allocated":630784,"data_stored":263223,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Jul 15 06:49:39 2024","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.63900000000000001,"15min":0.58899999999999997},"min":{"1min":0.28499999999999998,"5min":0.28299999999999997,"15min":0.22700000000000001},"max":{"1min":0.93500000000000005,"5min":1.4690000000000001,"15min":1.4690000000000001},"last":0.57099999999999995},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.68200000000000005,"15min":0.59699999999999998},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.30099999999999999},"max":{"1min":0.95499999999999996,"5min":1.095,"15min":1.095},"last":0.87}]},{"osd":1,"last update":"Mon Jul 15 06:49:39 2024","interfaces":[{"interface":"back","average":{"1min":0.64500000000000002,"5min":0.70499999999999996,"15min":0.64400000000000002},"min":{"1min":0.34699999999999998,"5min":0.31900000000000001,"15min":0.28399999999999997},"max":{"1min":0.85799999999999998,"5min":1.177,"15min":1.177},"last":0.91600000000000004},{"interface":"front","average":{"1min":0.73099999999999998,"5min":0.76100000000000001,"15min":0.65100000000000002},"min":{"1min":0.40699999999999997,"5min":0.314,"15min":0.27400000000000002},"max":{"1min":1.224,"5min":1.2330000000000001,"15min":1.2330000000000001},"last":0.67800000000000005}]},{"osd":3,"last update":"Mon Jul 15 06:49:28 2024","interfaces":[{"interface":"back","average":{"1min":0.70999999999999996,"5min":0.76300000000000001,"15min":0.75600000000000001},"min":{"1min":0.375,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":1.03,"5min":1.256,"15min":1.256},"last":0.78800000000000003},{"interface":"front","average":{"1min":0.78500000000000003,"5min":0.81699999999999995,"15min":0.746},"min":{"1min":0.44800000000000001,"5min":0.433,"15min":0.433},"max":{"1min":1.1559999999999999,"5min":1.2549999999999999,"15min":1.2549999999999999},"last":0.76600000000000001}]},{"osd":4,"last update":"Mon Jul 15 06:49:10 2024","interfaces":[{"interface":"back","average":{"1min":0.98999999999999999,"5min":0.94599999999999995,"15min":0.89400000000000002},"min":{"1min":0.60899999999999999,"5min":0.56299999999999994,"15min":0.56299999999999994},"max":{"1min":1.3120000000000001,"5min":1.3120000000000001,"15min":1.3120000000000001},"last":0.98199999999999998},{"interface":"front","average":{"1min":1.0289999999999999,"5min":1.0089999999999999,"15min":0.94899999999999995},"min":{"1min":0.67900000000000005,"5min":0.50800000000000001,"15min":0.50800000000000001},"max":{"1min":1.3740000000000001,"5min":1.47,"15min":1.47},"last":1.0609999999999999}]},{"osd":5,"last update":"Mon Jul 15 06:49:10 2024","interfaces":[{"interface":"back","average":{"1min":1.042,"5min":0.997,"15min":0.96899999999999997},"min":{"1min":0.78000000000000003,"5min":0.48299999999999998,"15min":0.48299999999999998},"max":{"1min":1.415,"5min":1.571,"15min":1.571},"last":1.0109999999999999},{"interface":"front","average":{"1min":0.96399999999999997,"5min":0.97199999999999998,"15min":0.96099999999999997},"min":{"1min":0.71099999999999997,"5min":0.57699999999999996,"15min":0.57699999999999996},"max":{"1min":1.202,"5min":1.571,"15min":1.571},"last":1.028}]},{"osd":6,"last update":"Mon Jul 15 06:50:04 2024","interfaces":[{"interface":"back","average":{"1min":1.0569999999999999,"5min":1.0489999999999999,"15min":1.0640000000000001},"min":{"1min":0.61399999999999999,"5min":0.52800000000000002,"15min":0.52800000000000002},"max":{"1min":1.8009999999999999,"5min":1.8009999999999999,"15min":1.8009999999999999},"last":1.105},{"interface":"front","average":{"1min":0.95999999999999996,"5min":0.94899999999999995,"15min":0.95299999999999996},"min":{"1min":0.63400000000000001,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":1.2050000000000001,"5min":2.3500000000000001,"15min":2.3500000000000001},"last":0.82899999999999996}]},{"osd":7,"last update":"Mon Jul 15 06:49:51 2024","interfaces":[{"interface":"back","average":{"1min":0.90800000000000003,"5min":1.036,"15min":1.0580000000000001},"min":{"1min":0.58199999999999996,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":1.3100000000000001,"5min":1.6299999999999999,"15min":1.6299999999999999},"last":1.087},{"interface":"front","average":{"1min":1.0780000000000001,"5min":1.081,"15min":1.073},"min":{"1min":0.69599999999999995,"5min":0.50600000000000001,"15min":0.50600000000000001},"max":{"1min":1.756,"5min":1.756,"15min":1.756},"last":0.94899999999999995}]}]},{"osd":1,"up_from":13,"seq":55834574953,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297972,"kb_used_data":616,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447180,"statfs":{"total":95995035648,"available":95689912320,"internally_reserved":0,"allocated":630784,"data_stored":263223,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Jul 15 06:49:45 2024","interfaces":[{"interface":"back","average":{"1min":0.64300000000000002,"5min":0.67300000000000004,"15min":0.57399999999999995},"min":{"1min":0.36199999999999999,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":1.02,"5min":1.2250000000000001,"15min":1.2250000000000001},"last":0.77700000000000002},{"interface":"front","average":{"1min":0.73699999999999999,"5min":0.72599999999999998,"15min":0.60899999999999999},"min":{"1min":0.51600000000000001,"5min":0.32700000000000001,"15min":0.312},"max":{"1min":1.181,"5min":1.5609999999999999,"15min":1.5609999999999999},"last":0.56100000000000005}]},{"osd":2,"last update":"Mon Jul 15 06:49:30 2024","interfaces":[{"interface":"back","average":{"1min":0.81499999999999995,"5min":0.76900000000000002,"15min":0.67200000000000004},"min":{"1min":0.435,"5min":0.38600000000000001,"15min":0.28299999999999997},"max":{"1min":1.0720000000000001,"5min":1.4379999999999999,"15min":1.7509999999999999},"last":0.69699999999999995},{"interface":"front","average":{"1min":0.751,"5min":0.749,"15min":0.627},"min":{"1min":0.47699999999999998,"5min":0.30499999999999999,"15min":0.29599999999999999},"max":{"1min":1.137,"5min":1.47,"15min":1.47},"last":0.72999999999999998}]},{"osd":3,"last update":"Mon Jul 15 06:49:30 2024","interfaces":[{"interface":"back","average":{"1min":0.83099999999999996,"5min":0.80700000000000005,"15min":0.78100000000000003},"min":{"1min":0.47499999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":1.0549999999999999,"5min":1.3220000000000001,"15min":1.3220000000000001},"last":0.89900000000000002},{"interface":"front","average":{"1min":0.82199999999999995,"5min":0.75700000000000001,"15min":0.71999999999999997},"min":{"1min":0.53500000000000003,"5min":0.377,"15min":0.377},"max":{"1min":1.431,"5min":1.431,"15min":1.431},"last":1.0760000000000001}]},{"osd":4,"last update":"Mon Jul 15 06:49:12 2024","interfaces":[{"interface":"back","average":{"1min":1.0960000000000001,"5min":0.999,"15min":0.94499999999999995},"min":{"1min":0.73599999999999999,"5min":0.66200000000000003,"15min":0.66200000000000003},"max":{"1min":1.512,"5min":1.5840000000000001,"15min":1.5840000000000001},"last":0.97999999999999998},{"interface":"front","average":{"1min":0.998,"5min":0.94999999999999996,"15min":0.92200000000000004},"min":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"max":{"1min":1.3149999999999999,"5min":1.6839999999999999,"15min":1.6839999999999999},"last":0.91500000000000004}]},{"osd":5,"last update":"Mon Jul 15 06:49:06 2024","interfaces":[{"interface":"back","average":{"1min":1.002,"5min":1.0309999999999999,"15min":1.0589999999999999},"min":{"1min":0.65900000000000003,"5min":0.63,"15min":0.63},"max":{"1min":1.5109999999999999,"5min":4.431,"15min":4.431},"last":0.93600000000000005},{"interface":"front","average":{"1min":0.996,"5min":1.024,"15min":1.036},"min":{"1min":0.76900000000000002,"5min":0.75800000000000001,"15min":0.75800000000000001},"max":{"1min":1.363,"5min":3.0670000000000002,"15min":3.0670000000000002},"last":1.028}]},{"osd":6,"last update":"Mon Jul 15 06:49:59 2024","interfaces":[{"interface":"back","average":{"1min":1.006,"5min":1.0580000000000001,"15min":1.0700000000000001},"min":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"max":{"1min":1.405,"5min":2.0339999999999998,"15min":2.0339999999999998},"last":0.85899999999999999},{"interface":"front","average":{"1min":1.0229999999999999,"5min":1.0649999999999999,"15min":1.0900000000000001},"min":{"1min":0.78200000000000003,"5min":0.65200000000000002,"15min":0.65200000000000002},"max":{"1min":1.3560000000000001,"5min":1.897,"15min":1.897},"last":0.95899999999999996}]},{"osd":7,"last update":"Mon Jul 15 06:49:53 2024","interfaces":[{"interface":"back","average":{"1min":1.143,"5min":1.1279999999999999,"15min":1.141},"min":{"1min":0.84299999999999997,"5min":0.72299999999999998,"15min":0.72299999999999998},"max":{"1min":1.4630000000000001,"5min":1.988,"15min":1.988},"last":1.052},{"interface":"front","average":{"1min":1.044,"5min":1.054,"15min":1.0660000000000001},"min":{"1min":0.77800000000000002,"5min":0.64200000000000002,"15min":0.64200000000000002},"max":{"1min":1.498,"5min":1.7809999999999999,"15min":1.7809999999999999},"last":1.0049999999999999}]}]},{"osd":0,"up_from":8,"seq":34359738484,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298552,"kb_used_data":1196,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446600,"statfs":{"total":95995035648,"available":95689318400,"internally_reserved":0,"allocated":1224704,"data_stored":853591,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Jul 15 06:49:39 2024","interfaces":[{"interface":"back","average":{"1min":0.66400000000000003,"5min":0.67100000000000004,"15min":0.57099999999999995},"min":{"1min":0.53200000000000003,"5min":0.33900000000000002,"15min":0.26900000000000002},"max":{"1min":0.98199999999999998,"5min":1.2070000000000001,"15min":1.2070000000000001},"last":1.0469999999999999},{"interface":"front","average":{"1min":0.79200000000000004,"5min":0.73999999999999999,"15min":0.627},"min":{"1min":0.54500000000000004,"5min":0.39200000000000002,"15min":0.28499999999999998},"max":{"1min":1.0569999999999999,"5min":1.1180000000000001,"15min":1.2110000000000001},"last":0.873}]},{"osd":2,"last update":"Mon Jul 15 06:49:31 2024","interfaces":[{"interface":"back","average":{"1min":0.80700000000000005,"5min":0.751,"15min":0.67900000000000005},"min":{"1min":0.55000000000000004,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":1.1040000000000001,"5min":1.1040000000000001,"15min":1.1040000000000001},"last":0.69699999999999995},{"interface":"front","average":{"1min":0.72099999999999997,"5min":0.71799999999999997,"15min":0.67200000000000004},"min":{"1min":0.44900000000000001,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":1.079,"5min":1.54,"15min":1.54},"last":0.64300000000000002}]},{"osd":3,"last update":"Mon Jul 15 06:49:20 2024","interfaces":[{"interface":"back","average":{"1min":0.82999999999999996,"5min":0.80000000000000004,"15min":0.76900000000000002},"min":{"1min":0.56399999999999995,"5min":0.371,"15min":0.371},"max":{"1min":1.1679999999999999,"5min":1.7470000000000001,"15min":1.7470000000000001},"last":0.751},{"interface":"front","average":{"1min":0.79200000000000004,"5min":0.74299999999999999,"15min":0.71599999999999997},"min":{"1min":0.65200000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.97799999999999998,"5min":0.99399999999999999,"15min":0.99399999999999999},"last":0.78500000000000003}]},{"osd":4,"last update":"Mon Jul 15 06:49:16 2024","interfaces":[{"interface":"back","average":{"1min":0.99399999999999999,"5min":0.92500000000000004,"15min":0.89700000000000002},"min":{"1min":0.64000000000000001,"5min":0.51800000000000002,"15min":0.51800000000000002},"max":{"1min":1.272,"5min":1.2909999999999999,"15min":1.2909999999999999},"last":0.94499999999999995},{"interface":"front","average":{"1min":1.0820000000000001,"5min":0.99099999999999999,"15min":0.93100000000000005},"min":{"1min":0.82199999999999995,"5min":0.52800000000000002,"15min":0.52800000000000002},"max":{"1min":1.234,"5min":1.498,"15min":1.498},"last":0.89900000000000002}]},{"osd":5,"last update":"Mon Jul 15 06:50:04 2024","interfaces":[{"interface":"back","average":{"1min":0.92300000000000004,"5min":0.91700000000000004,"15min":0.94499999999999995},"min":{"1min":0.66300000000000003,"5min":0.58399999999999996,"15min":0.58399999999999996},"max":{"1min":1.167,"5min":1.3280000000000001,"15min":1.3280000000000001},"last":0.92100000000000004},{"interface":"front","average":{"1min":0.94699999999999995,"5min":0.95799999999999996,"15min":0.95599999999999996},"min":{"1min":0.71499999999999997,"5min":0.51000000000000001,"15min":0.51000000000000001},"max":{"1min":1.179,"5min":1.4350000000000001,"15min":1.4350000000000001},"last":0.97999999999999998}]},{"osd":6,"last update":"Mon Jul 15 06:49:59 2024","interfaces":[{"interface":"back","average":{"1min":1.0509999999999999,"5min":0.998,"15min":0.98199999999999998},"min":{"1min":0.55000000000000004,"5min":0.5,"15min":0.5},"max":{"1min":1.359,"5min":1.359,"15min":1.359},"last":1.0189999999999999},{"interface":"front","average":{"1min":0.997,"5min":1.03,"15min":1.0429999999999999},"min":{"1min":0.77200000000000002,"5min":0.47799999999999998,"15min":0.47799999999999998},"max":{"1min":1.2869999999999999,"5min":1.4039999999999999,"15min":1.4039999999999999},"last":1.1040000000000001}]},{"osd":7,"last update":"Mon Jul 15 06:49:50 2024","interfaces":[{"interface":"back","average":{"1min":1.091,"5min":1.0609999999999999,"15min":1.052},"min":{"1min":0.68500000000000005,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":1.3089999999999999,"5min":1.3200000000000001,"15min":1.3200000000000001},"last":1.079},{"interface":"front","average":{"1min":1.054,"5min":1.069,"15min":1.0549999999999999},"min":{"1min":0.93300000000000005,"5min":0.60999999999999999,"15min":0.60999999999999999},"max":{"1min":1.3640000000000001,"5min":1.827,"15min":1.827},"last":1.161}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-07-15T06:50:10.505 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-07-15T06:50:10.505 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-07-15T06:50:10.505 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-07-15T06:50:10.505 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph health --format=json 2024-07-15T06:50:12.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:11 smithi134 bash[24068]: audit 2024-07-15T06:50:09.854088+0000 mgr.y (mgr.24272) 107 : audit [DBG] from='client.14550 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-07-15T06:50:12.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:11 smithi134 bash[24068]: cluster 2024-07-15T06:50:10.145443+0000 mgr.y (mgr.24272) 108 : cluster [DBG] pgmap v80: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:50:12.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:11 smithi089 bash[21793]: audit 2024-07-15T06:50:09.854088+0000 mgr.y (mgr.24272) 107 : audit [DBG] from='client.14550 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-07-15T06:50:12.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:11 smithi089 bash[21793]: cluster 2024-07-15T06:50:10.145443+0000 mgr.y (mgr.24272) 108 : cluster [DBG] pgmap v80: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:50:12.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:11 smithi089 bash[31372]: audit 2024-07-15T06:50:09.854088+0000 mgr.y (mgr.24272) 107 : audit [DBG] from='client.14550 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-07-15T06:50:12.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:11 smithi089 bash[31372]: cluster 2024-07-15T06:50:10.145443+0000 mgr.y (mgr.24272) 108 : cluster [DBG] pgmap v80: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:50:13.119 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:50:12 smithi089 bash[22045]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:50:12] "GET /metrics HTTP/1.1" 200 203246 "" "Prometheus/2.43.0" 2024-07-15T06:50:14.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:13 smithi134 bash[24068]: cluster 2024-07-15T06:50:12.145988+0000 mgr.y (mgr.24272) 109 : cluster [DBG] pgmap v81: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:50:14.118 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:13 smithi089 bash[21793]: cluster 2024-07-15T06:50:12.145988+0000 mgr.y (mgr.24272) 109 : cluster [DBG] pgmap v81: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:50:14.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:13 smithi089 bash[31372]: cluster 2024-07-15T06:50:12.145988+0000 mgr.y (mgr.24272) 109 : cluster [DBG] pgmap v81: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:50:15.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:14 smithi134 bash[24068]: audit 2024-07-15T06:50:14.191017+0000 mon.a (mon.0) 793 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:50:15.118 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:14 smithi089 bash[21793]: audit 2024-07-15T06:50:14.191017+0000 mon.a (mon.0) 793 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:50:15.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:14 smithi089 bash[31372]: audit 2024-07-15T06:50:14.191017+0000 mon.a (mon.0) 793 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:50:15.273 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T06:50:16.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:15 smithi134 bash[24068]: cluster 2024-07-15T06:50:14.146552+0000 mgr.y (mgr.24272) 110 : cluster [DBG] pgmap v82: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:50:16.118 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:15 smithi089 bash[21793]: cluster 2024-07-15T06:50:14.146552+0000 mgr.y (mgr.24272) 110 : cluster [DBG] pgmap v82: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:50:16.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:15 smithi089 bash[31372]: cluster 2024-07-15T06:50:14.146552+0000 mgr.y (mgr.24272) 110 : cluster [DBG] pgmap v82: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:50:17.304 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:50:16 smithi134 bash[25019]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:50:16] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-07-15T06:50:18.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:17 smithi134 bash[24068]: cluster 2024-07-15T06:50:16.147315+0000 mgr.y (mgr.24272) 111 : cluster [DBG] pgmap v83: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:50:18.118 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:17 smithi089 bash[21793]: cluster 2024-07-15T06:50:16.147315+0000 mgr.y (mgr.24272) 111 : cluster [DBG] pgmap v83: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:50:18.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:17 smithi089 bash[31372]: cluster 2024-07-15T06:50:16.147315+0000 mgr.y (mgr.24272) 111 : cluster [DBG] pgmap v83: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:50:18.691 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-07-15T06:50:18.691 INFO:teuthology.orchestra.run.smithi089.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-07-15T06:50:19.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:18 smithi134 bash[24068]: audit 2024-07-15T06:50:18.697360+0000 mon.c (mon.2) 24 : audit [DBG] from='client.? 172.21.15.89:0/1942449677' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-15T06:50:19.118 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:18 smithi089 bash[21793]: audit 2024-07-15T06:50:18.697360+0000 mon.c (mon.2) 24 : audit [DBG] from='client.? 172.21.15.89:0/1942449677' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-15T06:50:19.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:18 smithi089 bash[31372]: audit 2024-07-15T06:50:18.697360+0000 mon.c (mon.2) 24 : audit [DBG] from='client.? 172.21.15.89:0/1942449677' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-15T06:50:19.369 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-07-15T06:50:19.370 INFO:tasks.cephadm:Setup complete, yielding 2024-07-15T06:50:19.370 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-07-15T06:50:19.380 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi089.front.sepia.ceph.com 2024-07-15T06:50:19.381 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- bash -c 'ceph fs volume create foo' 2024-07-15T06:50:20.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:19 smithi134 bash[24068]: cluster 2024-07-15T06:50:18.147912+0000 mgr.y (mgr.24272) 112 : cluster [DBG] pgmap v84: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:50:20.118 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:19 smithi089 bash[21793]: cluster 2024-07-15T06:50:18.147912+0000 mgr.y (mgr.24272) 112 : cluster [DBG] pgmap v84: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:50:20.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:19 smithi089 bash[31372]: cluster 2024-07-15T06:50:18.147912+0000 mgr.y (mgr.24272) 112 : cluster [DBG] pgmap v84: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:50:21.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:20 smithi134 bash[24068]: cluster 2024-07-15T06:50:20.148512+0000 mgr.y (mgr.24272) 113 : cluster [DBG] pgmap v85: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:50:21.118 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:20 smithi089 bash[21793]: cluster 2024-07-15T06:50:20.148512+0000 mgr.y (mgr.24272) 113 : cluster [DBG] pgmap v85: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:50:21.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:20 smithi089 bash[31372]: cluster 2024-07-15T06:50:20.148512+0000 mgr.y (mgr.24272) 113 : cluster [DBG] pgmap v85: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:50:23.118 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:50:22 smithi089 bash[22045]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:50:22] "GET /metrics HTTP/1.1" 200 203248 "" "Prometheus/2.43.0" 2024-07-15T06:50:23.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:23 smithi134 bash[24068]: cluster 2024-07-15T06:50:22.148958+0000 mgr.y (mgr.24272) 114 : cluster [DBG] pgmap v86: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:50:23.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:23 smithi089 bash[21793]: cluster 2024-07-15T06:50:22.148958+0000 mgr.y (mgr.24272) 114 : cluster [DBG] pgmap v86: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:50:23.619 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:23 smithi089 bash[31372]: cluster 2024-07-15T06:50:22.148958+0000 mgr.y (mgr.24272) 114 : cluster [DBG] pgmap v86: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:50:24.161 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T06:50:25.497 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:25 smithi089 bash[31372]: cluster 2024-07-15T06:50:24.149518+0000 mgr.y (mgr.24272) 115 : cluster [DBG] pgmap v87: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:50:25.498 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:25 smithi089 bash[21793]: cluster 2024-07-15T06:50:24.149518+0000 mgr.y (mgr.24272) 115 : cluster [DBG] pgmap v87: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:50:25.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:25 smithi134 bash[24068]: cluster 2024-07-15T06:50:24.149518+0000 mgr.y (mgr.24272) 115 : cluster [DBG] pgmap v87: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:50:27.304 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:50:26 smithi134 bash[25019]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:50:26] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-07-15T06:50:27.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:27 smithi134 bash[24068]: cluster 2024-07-15T06:50:26.150198+0000 mgr.y (mgr.24272) 116 : cluster [DBG] pgmap v88: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:50:27.618 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:27 smithi089 bash[21793]: cluster 2024-07-15T06:50:26.150198+0000 mgr.y (mgr.24272) 116 : cluster [DBG] pgmap v88: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:50:27.619 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:27 smithi089 bash[31372]: cluster 2024-07-15T06:50:26.150198+0000 mgr.y (mgr.24272) 116 : cluster [DBG] pgmap v88: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:50:28.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:28 smithi134 bash[24068]: audit 2024-07-15T06:50:27.653383+0000 mgr.y (mgr.24272) 117 : audit [DBG] from='client.14562 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T06:50:28.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:28 smithi134 bash[24068]: audit 2024-07-15T06:50:27.654210+0000 mon.a (mon.0) 794 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-07-15T06:50:28.618 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:28 smithi089 bash[21793]: audit 2024-07-15T06:50:27.653383+0000 mgr.y (mgr.24272) 117 : audit [DBG] from='client.14562 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T06:50:28.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:28 smithi089 bash[21793]: audit 2024-07-15T06:50:27.654210+0000 mon.a (mon.0) 794 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-07-15T06:50:28.619 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:28 smithi089 bash[31372]: audit 2024-07-15T06:50:27.653383+0000 mgr.y (mgr.24272) 117 : audit [DBG] from='client.14562 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T06:50:28.619 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:28 smithi089 bash[31372]: audit 2024-07-15T06:50:27.654210+0000 mon.a (mon.0) 794 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-07-15T06:50:29.514 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:29 smithi089 bash[21793]: cluster 2024-07-15T06:50:28.150717+0000 mgr.y (mgr.24272) 118 : cluster [DBG] pgmap v89: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:50:29.514 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:29 smithi089 bash[21793]: audit 2024-07-15T06:50:28.228533+0000 mon.a (mon.0) 795 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-07-15T06:50:29.514 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:29 smithi089 bash[21793]: cluster 2024-07-15T06:50:28.228670+0000 mon.a (mon.0) 796 : cluster [DBG] osdmap e64: 8 total, 8 up, 8 in 2024-07-15T06:50:29.514 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:29 smithi089 bash[21793]: audit 2024-07-15T06:50:28.237394+0000 mon.a (mon.0) 797 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-07-15T06:50:29.515 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:29 smithi089 bash[21793]: audit 2024-07-15T06:50:29.191567+0000 mon.a (mon.0) 798 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:50:29.515 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:29 smithi089 bash[21793]: debug 2024-07-15T06:50:29.281+0000 7fb0bbe15640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-07-15T06:50:29.515 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:29 smithi089 bash[31372]: cluster 2024-07-15T06:50:28.150717+0000 mgr.y (mgr.24272) 118 : cluster [DBG] pgmap v89: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:50:29.515 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:29 smithi089 bash[31372]: audit 2024-07-15T06:50:28.228533+0000 mon.a (mon.0) 795 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-07-15T06:50:29.515 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:29 smithi089 bash[31372]: cluster 2024-07-15T06:50:28.228670+0000 mon.a (mon.0) 796 : cluster [DBG] osdmap e64: 8 total, 8 up, 8 in 2024-07-15T06:50:29.515 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:29 smithi089 bash[31372]: audit 2024-07-15T06:50:28.237394+0000 mon.a (mon.0) 797 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-07-15T06:50:29.515 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:29 smithi089 bash[31372]: audit 2024-07-15T06:50:29.191567+0000 mon.a (mon.0) 798 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:50:29.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:29 smithi134 bash[24068]: cluster 2024-07-15T06:50:28.150717+0000 mgr.y (mgr.24272) 118 : cluster [DBG] pgmap v89: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:50:29.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:29 smithi134 bash[24068]: audit 2024-07-15T06:50:28.228533+0000 mon.a (mon.0) 795 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-07-15T06:50:29.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:29 smithi134 bash[24068]: cluster 2024-07-15T06:50:28.228670+0000 mon.a (mon.0) 796 : cluster [DBG] osdmap e64: 8 total, 8 up, 8 in 2024-07-15T06:50:29.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:29 smithi134 bash[24068]: audit 2024-07-15T06:50:28.237394+0000 mon.a (mon.0) 797 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-07-15T06:50:29.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:29 smithi134 bash[24068]: audit 2024-07-15T06:50:29.191567+0000 mon.a (mon.0) 798 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:50:30.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:30 smithi134 bash[24068]: audit 2024-07-15T06:50:29.234155+0000 mon.a (mon.0) 799 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-07-15T06:50:30.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:30 smithi134 bash[24068]: cluster 2024-07-15T06:50:29.234315+0000 mon.a (mon.0) 800 : cluster [DBG] osdmap e65: 8 total, 8 up, 8 in 2024-07-15T06:50:30.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:30 smithi134 bash[24068]: audit 2024-07-15T06:50:29.283356+0000 mon.a (mon.0) 801 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-07-15T06:50:30.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:30 smithi134 bash[24068]: cluster 2024-07-15T06:50:29.283912+0000 mon.a (mon.0) 802 : cluster [ERR] Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-07-15T06:50:30.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:30 smithi134 bash[24068]: cluster 2024-07-15T06:50:29.283930+0000 mon.a (mon.0) 803 : cluster [WRN] Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-07-15T06:50:30.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:30 smithi134 bash[24068]: cluster 2024-07-15T06:50:29.290732+0000 mon.a (mon.0) 804 : cluster [DBG] osdmap e66: 8 total, 8 up, 8 in 2024-07-15T06:50:30.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:30 smithi134 bash[24068]: audit 2024-07-15T06:50:29.292477+0000 mon.a (mon.0) 805 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-07-15T06:50:30.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:30 smithi134 bash[24068]: cluster 2024-07-15T06:50:29.292587+0000 mon.a (mon.0) 806 : cluster [DBG] fsmap foo:0 2024-07-15T06:50:30.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:30 smithi134 bash[24068]: cephadm 2024-07-15T06:50:29.297371+0000 mgr.y (mgr.24272) 119 : cephadm [INF] Saving service mds.foo spec with placement count:2 2024-07-15T06:50:30.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:30 smithi134 bash[24068]: audit 2024-07-15T06:50:29.305638+0000 mon.a (mon.0) 807 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:50:30.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:30 smithi134 bash[24068]: audit 2024-07-15T06:50:29.310197+0000 mon.a (mon.0) 808 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:50:30.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:30 smithi134 bash[24068]: audit 2024-07-15T06:50:29.313485+0000 mon.a (mon.0) 809 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:50:30.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:30 smithi134 bash[24068]: audit 2024-07-15T06:50:29.314392+0000 mon.a (mon.0) 810 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:50:30.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:30 smithi134 bash[24068]: audit 2024-07-15T06:50:29.319423+0000 mon.a (mon.0) 811 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:50:30.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:30 smithi134 bash[24068]: audit 2024-07-15T06:50:29.323620+0000 mon.a (mon.0) 812 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi089.cczuoz", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-07-15T06:50:30.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:30 smithi134 bash[24068]: audit 2024-07-15T06:50:29.328363+0000 mon.a (mon.0) 813 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi089.cczuoz", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-07-15T06:50:30.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:30 smithi134 bash[24068]: audit 2024-07-15T06:50:29.329321+0000 mon.a (mon.0) 814 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:50:30.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:30 smithi134 bash[24068]: cephadm 2024-07-15T06:50:29.330240+0000 mgr.y (mgr.24272) 120 : cephadm [INF] Deploying daemon mds.foo.smithi089.cczuoz on smithi089 2024-07-15T06:50:30.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:30 smithi089 bash[21793]: audit 2024-07-15T06:50:29.234155+0000 mon.a (mon.0) 799 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-07-15T06:50:30.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:30 smithi089 bash[21793]: cluster 2024-07-15T06:50:29.234315+0000 mon.a (mon.0) 800 : cluster [DBG] osdmap e65: 8 total, 8 up, 8 in 2024-07-15T06:50:30.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:30 smithi089 bash[21793]: audit 2024-07-15T06:50:29.283356+0000 mon.a (mon.0) 801 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-07-15T06:50:30.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:30 smithi089 bash[21793]: cluster 2024-07-15T06:50:29.283912+0000 mon.a (mon.0) 802 : cluster [ERR] Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-07-15T06:50:30.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:30 smithi089 bash[21793]: cluster 2024-07-15T06:50:29.283930+0000 mon.a (mon.0) 803 : cluster [WRN] Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-07-15T06:50:30.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:30 smithi089 bash[21793]: cluster 2024-07-15T06:50:29.290732+0000 mon.a (mon.0) 804 : cluster [DBG] osdmap e66: 8 total, 8 up, 8 in 2024-07-15T06:50:30.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:30 smithi089 bash[21793]: audit 2024-07-15T06:50:29.292477+0000 mon.a (mon.0) 805 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-07-15T06:50:30.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:30 smithi089 bash[21793]: cluster 2024-07-15T06:50:29.292587+0000 mon.a (mon.0) 806 : cluster [DBG] fsmap foo:0 2024-07-15T06:50:30.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:30 smithi089 bash[21793]: cephadm 2024-07-15T06:50:29.297371+0000 mgr.y (mgr.24272) 119 : cephadm [INF] Saving service mds.foo spec with placement count:2 2024-07-15T06:50:30.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:30 smithi089 bash[21793]: audit 2024-07-15T06:50:29.305638+0000 mon.a (mon.0) 807 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:50:30.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:30 smithi089 bash[21793]: audit 2024-07-15T06:50:29.310197+0000 mon.a (mon.0) 808 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:50:30.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:30 smithi089 bash[21793]: audit 2024-07-15T06:50:29.313485+0000 mon.a (mon.0) 809 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:50:30.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:30 smithi089 bash[21793]: audit 2024-07-15T06:50:29.314392+0000 mon.a (mon.0) 810 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:50:30.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:30 smithi089 bash[21793]: audit 2024-07-15T06:50:29.319423+0000 mon.a (mon.0) 811 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:50:30.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:30 smithi089 bash[21793]: audit 2024-07-15T06:50:29.323620+0000 mon.a (mon.0) 812 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi089.cczuoz", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-07-15T06:50:30.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:30 smithi089 bash[21793]: audit 2024-07-15T06:50:29.328363+0000 mon.a (mon.0) 813 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi089.cczuoz", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-07-15T06:50:30.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:30 smithi089 bash[21793]: audit 2024-07-15T06:50:29.329321+0000 mon.a (mon.0) 814 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:50:30.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:30 smithi089 bash[21793]: cephadm 2024-07-15T06:50:29.330240+0000 mgr.y (mgr.24272) 120 : cephadm [INF] Deploying daemon mds.foo.smithi089.cczuoz on smithi089 2024-07-15T06:50:30.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:30 smithi089 bash[31372]: audit 2024-07-15T06:50:29.234155+0000 mon.a (mon.0) 799 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-07-15T06:50:30.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:30 smithi089 bash[31372]: cluster 2024-07-15T06:50:29.234315+0000 mon.a (mon.0) 800 : cluster [DBG] osdmap e65: 8 total, 8 up, 8 in 2024-07-15T06:50:30.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:30 smithi089 bash[31372]: audit 2024-07-15T06:50:29.283356+0000 mon.a (mon.0) 801 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-07-15T06:50:30.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:30 smithi089 bash[31372]: cluster 2024-07-15T06:50:29.283912+0000 mon.a (mon.0) 802 : cluster [ERR] Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-07-15T06:50:30.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:30 smithi089 bash[31372]: cluster 2024-07-15T06:50:29.283930+0000 mon.a (mon.0) 803 : cluster [WRN] Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-07-15T06:50:30.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:30 smithi089 bash[31372]: cluster 2024-07-15T06:50:29.290732+0000 mon.a (mon.0) 804 : cluster [DBG] osdmap e66: 8 total, 8 up, 8 in 2024-07-15T06:50:30.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:30 smithi089 bash[31372]: audit 2024-07-15T06:50:29.292477+0000 mon.a (mon.0) 805 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-07-15T06:50:30.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:30 smithi089 bash[31372]: cluster 2024-07-15T06:50:29.292587+0000 mon.a (mon.0) 806 : cluster [DBG] fsmap foo:0 2024-07-15T06:50:30.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:30 smithi089 bash[31372]: cephadm 2024-07-15T06:50:29.297371+0000 mgr.y (mgr.24272) 119 : cephadm [INF] Saving service mds.foo spec with placement count:2 2024-07-15T06:50:30.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:30 smithi089 bash[31372]: audit 2024-07-15T06:50:29.305638+0000 mon.a (mon.0) 807 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:50:30.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:30 smithi089 bash[31372]: audit 2024-07-15T06:50:29.310197+0000 mon.a (mon.0) 808 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:50:30.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:30 smithi089 bash[31372]: audit 2024-07-15T06:50:29.313485+0000 mon.a (mon.0) 809 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:50:30.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:30 smithi089 bash[31372]: audit 2024-07-15T06:50:29.314392+0000 mon.a (mon.0) 810 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:50:30.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:30 smithi089 bash[31372]: audit 2024-07-15T06:50:29.319423+0000 mon.a (mon.0) 811 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:50:30.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:30 smithi089 bash[31372]: audit 2024-07-15T06:50:29.323620+0000 mon.a (mon.0) 812 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi089.cczuoz", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-07-15T06:50:30.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:30 smithi089 bash[31372]: audit 2024-07-15T06:50:29.328363+0000 mon.a (mon.0) 813 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi089.cczuoz", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-07-15T06:50:30.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:30 smithi089 bash[31372]: audit 2024-07-15T06:50:29.329321+0000 mon.a (mon.0) 814 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:50:30.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:30 smithi089 bash[31372]: cephadm 2024-07-15T06:50:29.330240+0000 mgr.y (mgr.24272) 120 : cephadm [INF] Deploying daemon mds.foo.smithi089.cczuoz on smithi089 2024-07-15T06:50:30.744 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-07-15T06:50:31.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:31 smithi134 bash[24068]: cluster 2024-07-15T06:50:30.151611+0000 mgr.y (mgr.24272) 121 : cluster [DBG] pgmap v93: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:50:31.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:31 smithi134 bash[24068]: cluster 2024-07-15T06:50:30.290741+0000 mon.a (mon.0) 815 : cluster [DBG] osdmap e67: 8 total, 8 up, 8 in 2024-07-15T06:50:31.616 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:31 smithi089 bash[21793]: cluster 2024-07-15T06:50:30.151611+0000 mgr.y (mgr.24272) 121 : cluster [DBG] pgmap v93: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:50:31.616 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:31 smithi089 bash[21793]: cluster 2024-07-15T06:50:30.290741+0000 mon.a (mon.0) 815 : cluster [DBG] osdmap e67: 8 total, 8 up, 8 in 2024-07-15T06:50:31.617 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:31 smithi089 bash[31372]: cluster 2024-07-15T06:50:30.151611+0000 mgr.y (mgr.24272) 121 : cluster [DBG] pgmap v93: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:50:31.617 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:31 smithi089 bash[31372]: cluster 2024-07-15T06:50:30.290741+0000 mon.a (mon.0) 815 : cluster [DBG] osdmap e67: 8 total, 8 up, 8 in 2024-07-15T06:50:32.368 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:32 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:50:32.369 INFO:journalctl@ceph.osd.0.smithi089.stdout:Jul 15 06:50:32 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:50:32.369 INFO:journalctl@ceph.osd.1.smithi089.stdout:Jul 15 06:50:32 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:50:32.369 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:50:32 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:50:32.369 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:32 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:50:32.370 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 06:50:32 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:50:32.370 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:50:32 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:50:32.370 INFO:journalctl@ceph.osd.2.smithi089.stdout:Jul 15 06:50:32 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:50:32.371 INFO:journalctl@ceph.osd.3.smithi089.stdout:Jul 15 06:50:32 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:50:32.736 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:32 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:50:32.737 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:32 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:50:32.737 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:50:32 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:50:32.737 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:50:32 smithi089 bash[22045]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:50:32] "GET /metrics HTTP/1.1" 200 205539 "" "Prometheus/2.43.0" 2024-07-15T06:50:32.737 INFO:journalctl@ceph.osd.0.smithi089.stdout:Jul 15 06:50:32 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:50:32.738 INFO:journalctl@ceph.osd.1.smithi089.stdout:Jul 15 06:50:32 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:50:32.738 INFO:journalctl@ceph.osd.2.smithi089.stdout:Jul 15 06:50:32 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:50:32.738 INFO:journalctl@ceph.osd.3.smithi089.stdout:Jul 15 06:50:32 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:50:32.739 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:50:32 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:50:32.739 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 06:50:32 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:50:33.957 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:33 smithi134 bash[24068]: cluster 2024-07-15T06:50:32.152197+0000 mgr.y (mgr.24272) 122 : cluster [DBG] pgmap v95: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:50:33.957 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:33 smithi134 bash[24068]: audit 2024-07-15T06:50:32.670117+0000 mon.a (mon.0) 816 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:50:33.957 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:33 smithi134 bash[24068]: audit 2024-07-15T06:50:32.675639+0000 mon.a (mon.0) 817 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:50:33.957 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:33 smithi134 bash[24068]: audit 2024-07-15T06:50:32.679801+0000 mon.a (mon.0) 818 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:50:33.957 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:33 smithi134 bash[24068]: audit 2024-07-15T06:50:32.680362+0000 mon.a (mon.0) 819 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi134.wugtzh", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-07-15T06:50:33.958 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:33 smithi134 bash[24068]: audit 2024-07-15T06:50:32.685335+0000 mon.a (mon.0) 820 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi134.wugtzh", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-07-15T06:50:33.958 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:33 smithi134 bash[24068]: audit 2024-07-15T06:50:32.685994+0000 mon.a (mon.0) 821 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:50:33.958 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:33 smithi134 bash[24068]: cephadm 2024-07-15T06:50:32.686701+0000 mgr.y (mgr.24272) 123 : cephadm [INF] Deploying daemon mds.foo.smithi134.wugtzh on smithi134 2024-07-15T06:50:34.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:33 smithi089 bash[21793]: cluster 2024-07-15T06:50:32.152197+0000 mgr.y (mgr.24272) 122 : cluster [DBG] pgmap v95: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:50:34.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:33 smithi089 bash[21793]: audit 2024-07-15T06:50:32.670117+0000 mon.a (mon.0) 816 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:50:34.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:33 smithi089 bash[21793]: audit 2024-07-15T06:50:32.675639+0000 mon.a (mon.0) 817 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:50:34.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:33 smithi089 bash[21793]: audit 2024-07-15T06:50:32.679801+0000 mon.a (mon.0) 818 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:50:34.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:33 smithi089 bash[21793]: audit 2024-07-15T06:50:32.680362+0000 mon.a (mon.0) 819 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi134.wugtzh", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-07-15T06:50:34.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:33 smithi089 bash[21793]: audit 2024-07-15T06:50:32.685335+0000 mon.a (mon.0) 820 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi134.wugtzh", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-07-15T06:50:34.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:33 smithi089 bash[21793]: audit 2024-07-15T06:50:32.685994+0000 mon.a (mon.0) 821 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:50:34.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:33 smithi089 bash[21793]: cephadm 2024-07-15T06:50:32.686701+0000 mgr.y (mgr.24272) 123 : cephadm [INF] Deploying daemon mds.foo.smithi134.wugtzh on smithi134 2024-07-15T06:50:34.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:33 smithi089 bash[31372]: cluster 2024-07-15T06:50:32.152197+0000 mgr.y (mgr.24272) 122 : cluster [DBG] pgmap v95: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:50:34.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:33 smithi089 bash[31372]: audit 2024-07-15T06:50:32.670117+0000 mon.a (mon.0) 816 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:50:34.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:33 smithi089 bash[31372]: audit 2024-07-15T06:50:32.675639+0000 mon.a (mon.0) 817 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:50:34.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:33 smithi089 bash[31372]: audit 2024-07-15T06:50:32.679801+0000 mon.a (mon.0) 818 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:50:34.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:33 smithi089 bash[31372]: audit 2024-07-15T06:50:32.680362+0000 mon.a (mon.0) 819 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi134.wugtzh", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-07-15T06:50:34.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:33 smithi089 bash[31372]: audit 2024-07-15T06:50:32.685335+0000 mon.a (mon.0) 820 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi134.wugtzh", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-07-15T06:50:34.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:33 smithi089 bash[31372]: audit 2024-07-15T06:50:32.685994+0000 mon.a (mon.0) 821 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:50:34.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:33 smithi089 bash[31372]: cephadm 2024-07-15T06:50:32.686701+0000 mgr.y (mgr.24272) 123 : cephadm [INF] Deploying daemon mds.foo.smithi134.wugtzh on smithi134 2024-07-15T06:50:34.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:34 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:50:34.994 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:50:34 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:50:34.995 INFO:journalctl@ceph.osd.4.smithi134.stdout:Jul 15 06:50:34 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:50:34.995 INFO:journalctl@ceph.osd.5.smithi134.stdout:Jul 15 06:50:34 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:50:34.995 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 06:50:34 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:50:34.995 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:50:34 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:50:34.996 INFO:journalctl@ceph.osd.6.smithi134.stdout:Jul 15 06:50:34 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:50:34.996 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:50:34 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:50:34.996 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:50:34 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:50:35.262 INFO:journalctl@ceph.osd.4.smithi134.stdout:Jul 15 06:50:35 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:50:35.262 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:50:35 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:50:35.263 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:50:35 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:50:35.263 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 06:50:35 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:50:35.264 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:50:35 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:50:35.264 INFO:journalctl@ceph.osd.5.smithi134.stdout:Jul 15 06:50:35 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:50:35.264 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:35 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:50:35.265 INFO:journalctl@ceph.osd.6.smithi134.stdout:Jul 15 06:50:35 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:50:35.265 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:50:35 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:50:35.524 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T06:50:35.869 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:35 smithi089 bash[21793]: cluster 2024-07-15T06:50:34.152816+0000 mgr.y (mgr.24272) 124 : cluster [DBG] pgmap v96: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:50:35.869 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:35 smithi089 bash[21793]: cluster 2024-07-15T06:50:34.683069+0000 mon.a (mon.0) 822 : cluster [DBG] mds.? [v2:172.21.15.89:6834/4041986456,v1:172.21.15.89:6835/4041986456] up:boot 2024-07-15T06:50:35.869 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:35 smithi089 bash[21793]: cluster 2024-07-15T06:50:34.683223+0000 mon.a (mon.0) 823 : cluster [INF] daemon mds.foo.smithi089.cczuoz assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-07-15T06:50:35.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:35 smithi089 bash[21793]: cluster 2024-07-15T06:50:34.683398+0000 mon.a (mon.0) 824 : cluster [INF] Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-07-15T06:50:35.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:35 smithi089 bash[21793]: cluster 2024-07-15T06:50:34.683435+0000 mon.a (mon.0) 825 : cluster [INF] Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-07-15T06:50:35.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:35 smithi089 bash[21793]: cluster 2024-07-15T06:50:34.683456+0000 mon.a (mon.0) 826 : cluster [INF] Cluster is now healthy 2024-07-15T06:50:35.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:35 smithi089 bash[21793]: cluster 2024-07-15T06:50:34.683944+0000 mon.a (mon.0) 827 : cluster [DBG] fsmap foo:0 1 up:standby 2024-07-15T06:50:35.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:35 smithi089 bash[21793]: audit 2024-07-15T06:50:34.684143+0000 mon.a (mon.0) 828 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi089.cczuoz"}]: dispatch 2024-07-15T06:50:35.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:35 smithi089 bash[21793]: cluster 2024-07-15T06:50:34.690864+0000 mon.a (mon.0) 829 : cluster [DBG] fsmap foo:1 {0=foo.smithi089.cczuoz=up:creating} 2024-07-15T06:50:35.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:35 smithi089 bash[21793]: cluster 2024-07-15T06:50:34.727459+0000 mon.a (mon.0) 830 : cluster [INF] daemon mds.foo.smithi089.cczuoz is now active in filesystem foo as rank 0 2024-07-15T06:50:35.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:35 smithi089 bash[21793]: audit 2024-07-15T06:50:35.306532+0000 mon.a (mon.0) 831 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:50:35.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:35 smithi089 bash[21793]: audit 2024-07-15T06:50:35.315893+0000 mon.a (mon.0) 832 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:50:35.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:35 smithi089 bash[21793]: audit 2024-07-15T06:50:35.323854+0000 mon.a (mon.0) 833 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:50:35.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:35 smithi089 bash[21793]: audit 2024-07-15T06:50:35.332723+0000 mon.a (mon.0) 834 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:50:35.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:35 smithi089 bash[21793]: audit 2024-07-15T06:50:35.340978+0000 mon.a (mon.0) 835 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:50:35.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:35 smithi089 bash[21793]: audit 2024-07-15T06:50:35.396626+0000 mon.a (mon.0) 836 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:50:35.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:35 smithi089 bash[31372]: cluster 2024-07-15T06:50:34.152816+0000 mgr.y (mgr.24272) 124 : cluster [DBG] pgmap v96: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:50:35.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:35 smithi089 bash[31372]: cluster 2024-07-15T06:50:34.683069+0000 mon.a (mon.0) 822 : cluster [DBG] mds.? [v2:172.21.15.89:6834/4041986456,v1:172.21.15.89:6835/4041986456] up:boot 2024-07-15T06:50:35.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:35 smithi089 bash[31372]: cluster 2024-07-15T06:50:34.683223+0000 mon.a (mon.0) 823 : cluster [INF] daemon mds.foo.smithi089.cczuoz assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-07-15T06:50:35.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:35 smithi089 bash[31372]: cluster 2024-07-15T06:50:34.683398+0000 mon.a (mon.0) 824 : cluster [INF] Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-07-15T06:50:35.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:35 smithi089 bash[31372]: cluster 2024-07-15T06:50:34.683435+0000 mon.a (mon.0) 825 : cluster [INF] Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-07-15T06:50:35.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:35 smithi089 bash[31372]: cluster 2024-07-15T06:50:34.683456+0000 mon.a (mon.0) 826 : cluster [INF] Cluster is now healthy 2024-07-15T06:50:35.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:35 smithi089 bash[31372]: cluster 2024-07-15T06:50:34.683944+0000 mon.a (mon.0) 827 : cluster [DBG] fsmap foo:0 1 up:standby 2024-07-15T06:50:35.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:35 smithi089 bash[31372]: audit 2024-07-15T06:50:34.684143+0000 mon.a (mon.0) 828 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi089.cczuoz"}]: dispatch 2024-07-15T06:50:35.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:35 smithi089 bash[31372]: cluster 2024-07-15T06:50:34.690864+0000 mon.a (mon.0) 829 : cluster [DBG] fsmap foo:1 {0=foo.smithi089.cczuoz=up:creating} 2024-07-15T06:50:35.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:35 smithi089 bash[31372]: cluster 2024-07-15T06:50:34.727459+0000 mon.a (mon.0) 830 : cluster [INF] daemon mds.foo.smithi089.cczuoz is now active in filesystem foo as rank 0 2024-07-15T06:50:35.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:35 smithi089 bash[31372]: audit 2024-07-15T06:50:35.306532+0000 mon.a (mon.0) 831 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:50:35.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:35 smithi089 bash[31372]: audit 2024-07-15T06:50:35.315893+0000 mon.a (mon.0) 832 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:50:35.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:35 smithi089 bash[31372]: audit 2024-07-15T06:50:35.323854+0000 mon.a (mon.0) 833 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:50:35.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:35 smithi089 bash[31372]: audit 2024-07-15T06:50:35.332723+0000 mon.a (mon.0) 834 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:50:35.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:35 smithi089 bash[31372]: audit 2024-07-15T06:50:35.340978+0000 mon.a (mon.0) 835 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:50:35.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:35 smithi089 bash[31372]: audit 2024-07-15T06:50:35.396626+0000 mon.a (mon.0) 836 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:50:36.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:35 smithi134 bash[24068]: cluster 2024-07-15T06:50:34.152816+0000 mgr.y (mgr.24272) 124 : cluster [DBG] pgmap v96: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:50:36.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:35 smithi134 bash[24068]: cluster 2024-07-15T06:50:34.683069+0000 mon.a (mon.0) 822 : cluster [DBG] mds.? [v2:172.21.15.89:6834/4041986456,v1:172.21.15.89:6835/4041986456] up:boot 2024-07-15T06:50:36.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:35 smithi134 bash[24068]: cluster 2024-07-15T06:50:34.683223+0000 mon.a (mon.0) 823 : cluster [INF] daemon mds.foo.smithi089.cczuoz assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-07-15T06:50:36.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:35 smithi134 bash[24068]: cluster 2024-07-15T06:50:34.683398+0000 mon.a (mon.0) 824 : cluster [INF] Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-07-15T06:50:36.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:35 smithi134 bash[24068]: cluster 2024-07-15T06:50:34.683435+0000 mon.a (mon.0) 825 : cluster [INF] Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-07-15T06:50:36.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:35 smithi134 bash[24068]: cluster 2024-07-15T06:50:34.683456+0000 mon.a (mon.0) 826 : cluster [INF] Cluster is now healthy 2024-07-15T06:50:36.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:35 smithi134 bash[24068]: cluster 2024-07-15T06:50:34.683944+0000 mon.a (mon.0) 827 : cluster [DBG] fsmap foo:0 1 up:standby 2024-07-15T06:50:36.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:35 smithi134 bash[24068]: audit 2024-07-15T06:50:34.684143+0000 mon.a (mon.0) 828 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi089.cczuoz"}]: dispatch 2024-07-15T06:50:36.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:35 smithi134 bash[24068]: cluster 2024-07-15T06:50:34.690864+0000 mon.a (mon.0) 829 : cluster [DBG] fsmap foo:1 {0=foo.smithi089.cczuoz=up:creating} 2024-07-15T06:50:36.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:35 smithi134 bash[24068]: cluster 2024-07-15T06:50:34.727459+0000 mon.a (mon.0) 830 : cluster [INF] daemon mds.foo.smithi089.cczuoz is now active in filesystem foo as rank 0 2024-07-15T06:50:36.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:35 smithi134 bash[24068]: audit 2024-07-15T06:50:35.306532+0000 mon.a (mon.0) 831 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:50:36.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:35 smithi134 bash[24068]: audit 2024-07-15T06:50:35.315893+0000 mon.a (mon.0) 832 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:50:36.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:35 smithi134 bash[24068]: audit 2024-07-15T06:50:35.323854+0000 mon.a (mon.0) 833 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:50:36.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:35 smithi134 bash[24068]: audit 2024-07-15T06:50:35.332723+0000 mon.a (mon.0) 834 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:50:36.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:35 smithi134 bash[24068]: audit 2024-07-15T06:50:35.340978+0000 mon.a (mon.0) 835 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:50:36.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:35 smithi134 bash[24068]: audit 2024-07-15T06:50:35.396626+0000 mon.a (mon.0) 836 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:50:36.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:36 smithi134 bash[24068]: cluster 2024-07-15T06:50:35.691122+0000 mon.a (mon.0) 837 : cluster [DBG] mds.? [v2:172.21.15.89:6834/4041986456,v1:172.21.15.89:6835/4041986456] up:active 2024-07-15T06:50:36.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:36 smithi134 bash[24068]: cluster 2024-07-15T06:50:35.691172+0000 mon.a (mon.0) 838 : cluster [DBG] fsmap foo:1 {0=foo.smithi089.cczuoz=up:active} 2024-07-15T06:50:37.080 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:36 smithi089 bash[21793]: cluster 2024-07-15T06:50:35.691122+0000 mon.a (mon.0) 837 : cluster [DBG] mds.? [v2:172.21.15.89:6834/4041986456,v1:172.21.15.89:6835/4041986456] up:active 2024-07-15T06:50:37.080 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:36 smithi089 bash[21793]: cluster 2024-07-15T06:50:35.691172+0000 mon.a (mon.0) 838 : cluster [DBG] fsmap foo:1 {0=foo.smithi089.cczuoz=up:active} 2024-07-15T06:50:37.081 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:36 smithi089 bash[31372]: cluster 2024-07-15T06:50:35.691122+0000 mon.a (mon.0) 837 : cluster [DBG] mds.? [v2:172.21.15.89:6834/4041986456,v1:172.21.15.89:6835/4041986456] up:active 2024-07-15T06:50:37.081 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:36 smithi089 bash[31372]: cluster 2024-07-15T06:50:35.691172+0000 mon.a (mon.0) 838 : cluster [DBG] fsmap foo:1 {0=foo.smithi089.cczuoz=up:active} 2024-07-15T06:50:37.304 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:50:36 smithi134 bash[25019]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:50:36] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-07-15T06:50:38.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:37 smithi134 bash[24068]: cluster 2024-07-15T06:50:36.153923+0000 mgr.y (mgr.24272) 125 : cluster [DBG] pgmap v97: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.9 KiB/s wr, 6 op/s 2024-07-15T06:50:38.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:37 smithi134 bash[24068]: cluster 2024-07-15T06:50:36.703659+0000 mon.a (mon.0) 839 : cluster [DBG] mds.? [v2:172.21.15.134:6832/968641534,v1:172.21.15.134:6833/968641534] up:boot 2024-07-15T06:50:38.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:37 smithi134 bash[24068]: cluster 2024-07-15T06:50:36.703761+0000 mon.a (mon.0) 840 : cluster [INF] Dropping low affinity active daemon mds.foo.smithi089.cczuoz in favor of higher affinity standby. 2024-07-15T06:50:38.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:37 smithi134 bash[24068]: cluster 2024-07-15T06:50:36.703835+0000 mon.a (mon.0) 841 : cluster [WRN] Replacing daemon mds.foo.smithi089.cczuoz as rank 0 with standby daemon mds.foo.smithi134.wugtzh 2024-07-15T06:50:38.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:37 smithi134 bash[24068]: cluster 2024-07-15T06:50:36.704467+0000 mon.a (mon.0) 842 : cluster [WRN] Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-07-15T06:50:38.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:37 smithi134 bash[24068]: cluster 2024-07-15T06:50:36.704507+0000 mon.a (mon.0) 843 : cluster [WRN] Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-07-15T06:50:38.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:37 smithi134 bash[24068]: cluster 2024-07-15T06:50:36.704982+0000 mon.a (mon.0) 844 : cluster [DBG] fsmap foo:1 {0=foo.smithi089.cczuoz=up:active} 1 up:standby 2024-07-15T06:50:38.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:37 smithi134 bash[24068]: audit 2024-07-15T06:50:36.705331+0000 mon.a (mon.0) 845 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi134.wugtzh"}]: dispatch 2024-07-15T06:50:38.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:37 smithi134 bash[24068]: cluster 2024-07-15T06:50:36.712763+0000 mon.a (mon.0) 846 : cluster [DBG] osdmap e68: 8 total, 8 up, 8 in 2024-07-15T06:50:38.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:37 smithi134 bash[24068]: cluster 2024-07-15T06:50:36.712989+0000 mon.a (mon.0) 847 : cluster [DBG] fsmap foo:1/1 {0=foo.smithi134.wugtzh=up:replay} 2024-07-15T06:50:38.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:37 smithi089 bash[21793]: cluster 2024-07-15T06:50:36.153923+0000 mgr.y (mgr.24272) 125 : cluster [DBG] pgmap v97: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.9 KiB/s wr, 6 op/s 2024-07-15T06:50:38.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:37 smithi089 bash[21793]: cluster 2024-07-15T06:50:36.703659+0000 mon.a (mon.0) 839 : cluster [DBG] mds.? [v2:172.21.15.134:6832/968641534,v1:172.21.15.134:6833/968641534] up:boot 2024-07-15T06:50:38.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:37 smithi089 bash[21793]: cluster 2024-07-15T06:50:36.703761+0000 mon.a (mon.0) 840 : cluster [INF] Dropping low affinity active daemon mds.foo.smithi089.cczuoz in favor of higher affinity standby. 2024-07-15T06:50:38.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:37 smithi089 bash[21793]: cluster 2024-07-15T06:50:36.703835+0000 mon.a (mon.0) 841 : cluster [WRN] Replacing daemon mds.foo.smithi089.cczuoz as rank 0 with standby daemon mds.foo.smithi134.wugtzh 2024-07-15T06:50:38.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:37 smithi089 bash[21793]: cluster 2024-07-15T06:50:36.704467+0000 mon.a (mon.0) 842 : cluster [WRN] Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-07-15T06:50:38.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:37 smithi089 bash[21793]: cluster 2024-07-15T06:50:36.704507+0000 mon.a (mon.0) 843 : cluster [WRN] Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-07-15T06:50:38.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:37 smithi089 bash[21793]: cluster 2024-07-15T06:50:36.704982+0000 mon.a (mon.0) 844 : cluster [DBG] fsmap foo:1 {0=foo.smithi089.cczuoz=up:active} 1 up:standby 2024-07-15T06:50:38.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:37 smithi089 bash[21793]: audit 2024-07-15T06:50:36.705331+0000 mon.a (mon.0) 845 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi134.wugtzh"}]: dispatch 2024-07-15T06:50:38.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:37 smithi089 bash[21793]: cluster 2024-07-15T06:50:36.712763+0000 mon.a (mon.0) 846 : cluster [DBG] osdmap e68: 8 total, 8 up, 8 in 2024-07-15T06:50:38.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:37 smithi089 bash[21793]: cluster 2024-07-15T06:50:36.712989+0000 mon.a (mon.0) 847 : cluster [DBG] fsmap foo:1/1 {0=foo.smithi134.wugtzh=up:replay} 2024-07-15T06:50:38.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:37 smithi089 bash[31372]: cluster 2024-07-15T06:50:36.153923+0000 mgr.y (mgr.24272) 125 : cluster [DBG] pgmap v97: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.9 KiB/s wr, 6 op/s 2024-07-15T06:50:38.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:37 smithi089 bash[31372]: cluster 2024-07-15T06:50:36.703659+0000 mon.a (mon.0) 839 : cluster [DBG] mds.? [v2:172.21.15.134:6832/968641534,v1:172.21.15.134:6833/968641534] up:boot 2024-07-15T06:50:38.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:37 smithi089 bash[31372]: cluster 2024-07-15T06:50:36.703761+0000 mon.a (mon.0) 840 : cluster [INF] Dropping low affinity active daemon mds.foo.smithi089.cczuoz in favor of higher affinity standby. 2024-07-15T06:50:38.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:37 smithi089 bash[31372]: cluster 2024-07-15T06:50:36.703835+0000 mon.a (mon.0) 841 : cluster [WRN] Replacing daemon mds.foo.smithi089.cczuoz as rank 0 with standby daemon mds.foo.smithi134.wugtzh 2024-07-15T06:50:38.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:37 smithi089 bash[31372]: cluster 2024-07-15T06:50:36.704467+0000 mon.a (mon.0) 842 : cluster [WRN] Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-07-15T06:50:38.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:37 smithi089 bash[31372]: cluster 2024-07-15T06:50:36.704507+0000 mon.a (mon.0) 843 : cluster [WRN] Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-07-15T06:50:38.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:37 smithi089 bash[31372]: cluster 2024-07-15T06:50:36.704982+0000 mon.a (mon.0) 844 : cluster [DBG] fsmap foo:1 {0=foo.smithi089.cczuoz=up:active} 1 up:standby 2024-07-15T06:50:38.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:37 smithi089 bash[31372]: audit 2024-07-15T06:50:36.705331+0000 mon.a (mon.0) 845 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi134.wugtzh"}]: dispatch 2024-07-15T06:50:38.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:37 smithi089 bash[31372]: cluster 2024-07-15T06:50:36.712763+0000 mon.a (mon.0) 846 : cluster [DBG] osdmap e68: 8 total, 8 up, 8 in 2024-07-15T06:50:38.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:37 smithi089 bash[31372]: cluster 2024-07-15T06:50:36.712989+0000 mon.a (mon.0) 847 : cluster [DBG] fsmap foo:1/1 {0=foo.smithi134.wugtzh=up:replay} 2024-07-15T06:50:39.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:38 smithi134 bash[24068]: cluster 2024-07-15T06:50:37.708218+0000 mon.a (mon.0) 848 : cluster [INF] Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-07-15T06:50:39.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:38 smithi134 bash[24068]: cluster 2024-07-15T06:50:37.715970+0000 mon.a (mon.0) 849 : cluster [DBG] mds.? [v2:172.21.15.134:6832/968641534,v1:172.21.15.134:6833/968641534] up:reconnect 2024-07-15T06:50:39.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:38 smithi134 bash[24068]: cluster 2024-07-15T06:50:37.716090+0000 mon.a (mon.0) 850 : cluster [DBG] mds.? [v2:172.21.15.89:6834/1598068998,v1:172.21.15.89:6835/1598068998] up:boot 2024-07-15T06:50:39.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:38 smithi134 bash[24068]: cluster 2024-07-15T06:50:37.716249+0000 mon.a (mon.0) 851 : cluster [DBG] fsmap foo:1/1 {0=foo.smithi134.wugtzh=up:reconnect} 1 up:standby 2024-07-15T06:50:39.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:38 smithi134 bash[24068]: audit 2024-07-15T06:50:37.716446+0000 mon.a (mon.0) 852 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi089.cczuoz"}]: dispatch 2024-07-15T06:50:39.118 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:38 smithi089 bash[21793]: cluster 2024-07-15T06:50:37.708218+0000 mon.a (mon.0) 848 : cluster [INF] Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-07-15T06:50:39.118 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:38 smithi089 bash[21793]: cluster 2024-07-15T06:50:37.715970+0000 mon.a (mon.0) 849 : cluster [DBG] mds.? [v2:172.21.15.134:6832/968641534,v1:172.21.15.134:6833/968641534] up:reconnect 2024-07-15T06:50:39.118 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:38 smithi089 bash[21793]: cluster 2024-07-15T06:50:37.716090+0000 mon.a (mon.0) 850 : cluster [DBG] mds.? [v2:172.21.15.89:6834/1598068998,v1:172.21.15.89:6835/1598068998] up:boot 2024-07-15T06:50:39.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:38 smithi089 bash[21793]: cluster 2024-07-15T06:50:37.716249+0000 mon.a (mon.0) 851 : cluster [DBG] fsmap foo:1/1 {0=foo.smithi134.wugtzh=up:reconnect} 1 up:standby 2024-07-15T06:50:39.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:38 smithi089 bash[21793]: audit 2024-07-15T06:50:37.716446+0000 mon.a (mon.0) 852 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi089.cczuoz"}]: dispatch 2024-07-15T06:50:39.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:38 smithi089 bash[31372]: cluster 2024-07-15T06:50:37.708218+0000 mon.a (mon.0) 848 : cluster [INF] Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-07-15T06:50:39.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:38 smithi089 bash[31372]: cluster 2024-07-15T06:50:37.715970+0000 mon.a (mon.0) 849 : cluster [DBG] mds.? [v2:172.21.15.134:6832/968641534,v1:172.21.15.134:6833/968641534] up:reconnect 2024-07-15T06:50:39.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:38 smithi089 bash[31372]: cluster 2024-07-15T06:50:37.716090+0000 mon.a (mon.0) 850 : cluster [DBG] mds.? [v2:172.21.15.89:6834/1598068998,v1:172.21.15.89:6835/1598068998] up:boot 2024-07-15T06:50:39.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:38 smithi089 bash[31372]: cluster 2024-07-15T06:50:37.716249+0000 mon.a (mon.0) 851 : cluster [DBG] fsmap foo:1/1 {0=foo.smithi134.wugtzh=up:reconnect} 1 up:standby 2024-07-15T06:50:39.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:38 smithi089 bash[31372]: audit 2024-07-15T06:50:37.716446+0000 mon.a (mon.0) 852 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi089.cczuoz"}]: dispatch 2024-07-15T06:50:39.798 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-07-15T06:50:40.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:39 smithi134 bash[24068]: cluster 2024-07-15T06:50:38.154609+0000 mgr.y (mgr.24272) 126 : cluster [DBG] pgmap v99: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.6 KiB/s wr, 5 op/s 2024-07-15T06:50:40.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:39 smithi134 bash[24068]: cluster 2024-07-15T06:50:38.726073+0000 mon.a (mon.0) 853 : cluster [DBG] mds.? [v2:172.21.15.134:6832/968641534,v1:172.21.15.134:6833/968641534] up:rejoin 2024-07-15T06:50:40.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:39 smithi134 bash[24068]: cluster 2024-07-15T06:50:38.726231+0000 mon.a (mon.0) 854 : cluster [DBG] fsmap foo:1/1 {0=foo.smithi134.wugtzh=up:rejoin} 1 up:standby 2024-07-15T06:50:40.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:39 smithi134 bash[24068]: cluster 2024-07-15T06:50:38.726869+0000 mon.a (mon.0) 855 : cluster [INF] daemon mds.foo.smithi134.wugtzh is now active in filesystem foo as rank 0 2024-07-15T06:50:40.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:39 smithi134 bash[24068]: audit 2024-07-15T06:50:39.293431+0000 mon.a (mon.0) 856 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:50:40.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:39 smithi089 bash[21793]: cluster 2024-07-15T06:50:38.154609+0000 mgr.y (mgr.24272) 126 : cluster [DBG] pgmap v99: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.6 KiB/s wr, 5 op/s 2024-07-15T06:50:40.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:39 smithi089 bash[21793]: cluster 2024-07-15T06:50:38.726073+0000 mon.a (mon.0) 853 : cluster [DBG] mds.? [v2:172.21.15.134:6832/968641534,v1:172.21.15.134:6833/968641534] up:rejoin 2024-07-15T06:50:40.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:39 smithi089 bash[21793]: cluster 2024-07-15T06:50:38.726231+0000 mon.a (mon.0) 854 : cluster [DBG] fsmap foo:1/1 {0=foo.smithi134.wugtzh=up:rejoin} 1 up:standby 2024-07-15T06:50:40.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:39 smithi089 bash[21793]: cluster 2024-07-15T06:50:38.726869+0000 mon.a (mon.0) 855 : cluster [INF] daemon mds.foo.smithi134.wugtzh is now active in filesystem foo as rank 0 2024-07-15T06:50:40.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:39 smithi089 bash[21793]: audit 2024-07-15T06:50:39.293431+0000 mon.a (mon.0) 856 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:50:40.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:39 smithi089 bash[31372]: cluster 2024-07-15T06:50:38.154609+0000 mgr.y (mgr.24272) 126 : cluster [DBG] pgmap v99: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.6 KiB/s wr, 5 op/s 2024-07-15T06:50:40.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:39 smithi089 bash[31372]: cluster 2024-07-15T06:50:38.726073+0000 mon.a (mon.0) 853 : cluster [DBG] mds.? [v2:172.21.15.134:6832/968641534,v1:172.21.15.134:6833/968641534] up:rejoin 2024-07-15T06:50:40.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:39 smithi089 bash[31372]: cluster 2024-07-15T06:50:38.726231+0000 mon.a (mon.0) 854 : cluster [DBG] fsmap foo:1/1 {0=foo.smithi134.wugtzh=up:rejoin} 1 up:standby 2024-07-15T06:50:40.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:39 smithi089 bash[31372]: cluster 2024-07-15T06:50:38.726869+0000 mon.a (mon.0) 855 : cluster [INF] daemon mds.foo.smithi134.wugtzh is now active in filesystem foo as rank 0 2024-07-15T06:50:40.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:39 smithi089 bash[31372]: audit 2024-07-15T06:50:39.293431+0000 mon.a (mon.0) 856 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:50:41.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:40 smithi134 bash[24068]: cluster 2024-07-15T06:50:39.720126+0000 mon.a (mon.0) 857 : cluster [INF] Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-07-15T06:50:41.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:40 smithi134 bash[24068]: cluster 2024-07-15T06:50:39.720171+0000 mon.a (mon.0) 858 : cluster [INF] Cluster is now healthy 2024-07-15T06:50:41.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:40 smithi134 bash[24068]: cluster 2024-07-15T06:50:39.729288+0000 mon.a (mon.0) 859 : cluster [DBG] mds.? [v2:172.21.15.134:6832/968641534,v1:172.21.15.134:6833/968641534] up:active 2024-07-15T06:50:41.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:40 smithi134 bash[24068]: cluster 2024-07-15T06:50:39.729430+0000 mon.a (mon.0) 860 : cluster [DBG] fsmap foo:1 {0=foo.smithi134.wugtzh=up:active} 1 up:standby 2024-07-15T06:50:41.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:40 smithi089 bash[21793]: cluster 2024-07-15T06:50:39.720126+0000 mon.a (mon.0) 857 : cluster [INF] Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-07-15T06:50:41.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:40 smithi089 bash[21793]: cluster 2024-07-15T06:50:39.720171+0000 mon.a (mon.0) 858 : cluster [INF] Cluster is now healthy 2024-07-15T06:50:41.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:40 smithi089 bash[21793]: cluster 2024-07-15T06:50:39.729288+0000 mon.a (mon.0) 859 : cluster [DBG] mds.? [v2:172.21.15.134:6832/968641534,v1:172.21.15.134:6833/968641534] up:active 2024-07-15T06:50:41.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:40 smithi089 bash[21793]: cluster 2024-07-15T06:50:39.729430+0000 mon.a (mon.0) 860 : cluster [DBG] fsmap foo:1 {0=foo.smithi134.wugtzh=up:active} 1 up:standby 2024-07-15T06:50:41.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:40 smithi089 bash[31372]: cluster 2024-07-15T06:50:39.720126+0000 mon.a (mon.0) 857 : cluster [INF] Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-07-15T06:50:41.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:40 smithi089 bash[31372]: cluster 2024-07-15T06:50:39.720171+0000 mon.a (mon.0) 858 : cluster [INF] Cluster is now healthy 2024-07-15T06:50:41.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:40 smithi089 bash[31372]: cluster 2024-07-15T06:50:39.729288+0000 mon.a (mon.0) 859 : cluster [DBG] mds.? [v2:172.21.15.134:6832/968641534,v1:172.21.15.134:6833/968641534] up:active 2024-07-15T06:50:41.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:40 smithi089 bash[31372]: cluster 2024-07-15T06:50:39.729430+0000 mon.a (mon.0) 860 : cluster [DBG] fsmap foo:1 {0=foo.smithi134.wugtzh=up:active} 1 up:standby 2024-07-15T06:50:42.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:41 smithi134 bash[24068]: cluster 2024-07-15T06:50:40.155657+0000 mgr.y (mgr.24272) 127 : cluster [DBG] pgmap v100: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 5.0 KiB/s rd, 1.3 KiB/s wr, 9 op/s 2024-07-15T06:50:42.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:41 smithi134 bash[24068]: audit 2024-07-15T06:50:40.774502+0000 mon.a (mon.0) 861 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:50:42.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:41 smithi134 bash[24068]: audit 2024-07-15T06:50:40.779278+0000 mon.a (mon.0) 862 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:50:42.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:41 smithi134 bash[24068]: audit 2024-07-15T06:50:41.056362+0000 mon.a (mon.0) 863 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:50:42.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:41 smithi134 bash[24068]: audit 2024-07-15T06:50:41.069794+0000 mon.a (mon.0) 864 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:50:42.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:41 smithi134 bash[24068]: audit 2024-07-15T06:50:41.071931+0000 mon.a (mon.0) 865 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:50:42.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:41 smithi134 bash[24068]: audit 2024-07-15T06:50:41.073920+0000 mon.a (mon.0) 866 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:50:42.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:41 smithi134 bash[24068]: audit 2024-07-15T06:50:41.085970+0000 mon.a (mon.0) 867 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:50:42.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:41 smithi134 bash[24068]: audit 2024-07-15T06:50:41.148808+0000 mon.a (mon.0) 868 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:50:42.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:41 smithi134 bash[24068]: audit 2024-07-15T06:50:41.151026+0000 mon.a (mon.0) 869 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:50:42.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:41 smithi134 bash[24068]: audit 2024-07-15T06:50:41.152153+0000 mon.a (mon.0) 870 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:50:42.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:41 smithi134 bash[24068]: audit 2024-07-15T06:50:41.162552+0000 mon.a (mon.0) 871 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:50:42.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:41 smithi089 bash[21793]: cluster 2024-07-15T06:50:40.155657+0000 mgr.y (mgr.24272) 127 : cluster [DBG] pgmap v100: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 5.0 KiB/s rd, 1.3 KiB/s wr, 9 op/s 2024-07-15T06:50:42.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:41 smithi089 bash[21793]: audit 2024-07-15T06:50:40.774502+0000 mon.a (mon.0) 861 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:50:42.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:41 smithi089 bash[21793]: audit 2024-07-15T06:50:40.779278+0000 mon.a (mon.0) 862 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:50:42.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:41 smithi089 bash[21793]: audit 2024-07-15T06:50:41.056362+0000 mon.a (mon.0) 863 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:50:42.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:41 smithi089 bash[21793]: audit 2024-07-15T06:50:41.069794+0000 mon.a (mon.0) 864 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:50:42.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:41 smithi089 bash[21793]: audit 2024-07-15T06:50:41.071931+0000 mon.a (mon.0) 865 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:50:42.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:41 smithi089 bash[21793]: audit 2024-07-15T06:50:41.073920+0000 mon.a (mon.0) 866 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:50:42.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:41 smithi089 bash[21793]: audit 2024-07-15T06:50:41.085970+0000 mon.a (mon.0) 867 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:50:42.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:41 smithi089 bash[21793]: audit 2024-07-15T06:50:41.148808+0000 mon.a (mon.0) 868 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:50:42.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:41 smithi089 bash[21793]: audit 2024-07-15T06:50:41.151026+0000 mon.a (mon.0) 869 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:50:42.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:41 smithi089 bash[21793]: audit 2024-07-15T06:50:41.152153+0000 mon.a (mon.0) 870 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:50:42.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:41 smithi089 bash[21793]: audit 2024-07-15T06:50:41.162552+0000 mon.a (mon.0) 871 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:50:42.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:41 smithi089 bash[31372]: cluster 2024-07-15T06:50:40.155657+0000 mgr.y (mgr.24272) 127 : cluster [DBG] pgmap v100: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 5.0 KiB/s rd, 1.3 KiB/s wr, 9 op/s 2024-07-15T06:50:42.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:41 smithi089 bash[31372]: audit 2024-07-15T06:50:40.774502+0000 mon.a (mon.0) 861 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:50:42.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:41 smithi089 bash[31372]: audit 2024-07-15T06:50:40.779278+0000 mon.a (mon.0) 862 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:50:42.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:41 smithi089 bash[31372]: audit 2024-07-15T06:50:41.056362+0000 mon.a (mon.0) 863 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:50:42.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:41 smithi089 bash[31372]: audit 2024-07-15T06:50:41.069794+0000 mon.a (mon.0) 864 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:50:42.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:41 smithi089 bash[31372]: audit 2024-07-15T06:50:41.071931+0000 mon.a (mon.0) 865 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:50:42.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:41 smithi089 bash[31372]: audit 2024-07-15T06:50:41.073920+0000 mon.a (mon.0) 866 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:50:42.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:41 smithi089 bash[31372]: audit 2024-07-15T06:50:41.085970+0000 mon.a (mon.0) 867 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:50:42.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:41 smithi089 bash[31372]: audit 2024-07-15T06:50:41.148808+0000 mon.a (mon.0) 868 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:50:42.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:41 smithi089 bash[31372]: audit 2024-07-15T06:50:41.151026+0000 mon.a (mon.0) 869 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:50:42.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:41 smithi089 bash[31372]: audit 2024-07-15T06:50:41.152153+0000 mon.a (mon.0) 870 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:50:42.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:41 smithi089 bash[31372]: audit 2024-07-15T06:50:41.162552+0000 mon.a (mon.0) 871 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:50:43.119 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:50:42 smithi089 bash[22045]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:50:42] "GET /metrics HTTP/1.1" 200 205539 "" "Prometheus/2.43.0" 2024-07-15T06:50:43.690 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T06:50:44.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:43 smithi134 bash[24068]: cluster 2024-07-15T06:50:42.156237+0000 mgr.y (mgr.24272) 128 : cluster [DBG] pgmap v101: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.9 KiB/s rd, 1.3 KiB/s wr, 9 op/s 2024-07-15T06:50:44.118 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:43 smithi089 bash[21793]: cluster 2024-07-15T06:50:42.156237+0000 mgr.y (mgr.24272) 128 : cluster [DBG] pgmap v101: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.9 KiB/s rd, 1.3 KiB/s wr, 9 op/s 2024-07-15T06:50:44.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:43 smithi089 bash[31372]: cluster 2024-07-15T06:50:42.156237+0000 mgr.y (mgr.24272) 128 : cluster [DBG] pgmap v101: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.9 KiB/s rd, 1.3 KiB/s wr, 9 op/s 2024-07-15T06:50:45.037 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:44 smithi089 bash[21793]: audit 2024-07-15T06:50:44.145036+0000 mon.a (mon.0) 872 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2024-07-15T06:50:45.037 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:44 smithi089 bash[21793]: audit 2024-07-15T06:50:44.145492+0000 mon.a (mon.0) 873 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2024-07-15T06:50:45.037 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:44 smithi089 bash[21793]: audit 2024-07-15T06:50:44.145853+0000 mon.a (mon.0) 874 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2024-07-15T06:50:45.037 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:44 smithi089 bash[21793]: audit 2024-07-15T06:50:44.191875+0000 mon.a (mon.0) 875 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:50:45.037 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:44 smithi089 bash[31372]: audit 2024-07-15T06:50:44.145036+0000 mon.a (mon.0) 872 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2024-07-15T06:50:45.037 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:44 smithi089 bash[31372]: audit 2024-07-15T06:50:44.145492+0000 mon.a (mon.0) 873 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2024-07-15T06:50:45.037 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:44 smithi089 bash[31372]: audit 2024-07-15T06:50:44.145853+0000 mon.a (mon.0) 874 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2024-07-15T06:50:45.038 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:44 smithi089 bash[31372]: audit 2024-07-15T06:50:44.191875+0000 mon.a (mon.0) 875 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:50:45.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:44 smithi134 bash[24068]: audit 2024-07-15T06:50:44.145036+0000 mon.a (mon.0) 872 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2024-07-15T06:50:45.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:44 smithi134 bash[24068]: audit 2024-07-15T06:50:44.145492+0000 mon.a (mon.0) 873 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2024-07-15T06:50:45.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:44 smithi134 bash[24068]: audit 2024-07-15T06:50:44.145853+0000 mon.a (mon.0) 874 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2024-07-15T06:50:45.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:44 smithi134 bash[24068]: audit 2024-07-15T06:50:44.191875+0000 mon.a (mon.0) 875 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:50:46.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:45 smithi134 bash[24068]: cluster 2024-07-15T06:50:44.156822+0000 mgr.y (mgr.24272) 129 : cluster [DBG] pgmap v102: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.9 KiB/s rd, 1.3 KiB/s wr, 9 op/s 2024-07-15T06:50:46.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:45 smithi134 bash[24068]: audit 2024-07-15T06:50:44.795130+0000 mon.a (mon.0) 876 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]': finished 2024-07-15T06:50:46.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:45 smithi134 bash[24068]: audit 2024-07-15T06:50:44.795461+0000 mon.a (mon.0) 877 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]': finished 2024-07-15T06:50:46.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:45 smithi134 bash[24068]: audit 2024-07-15T06:50:44.795639+0000 mon.a (mon.0) 878 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]': finished 2024-07-15T06:50:46.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:45 smithi134 bash[24068]: cluster 2024-07-15T06:50:44.795773+0000 mon.a (mon.0) 879 : cluster [DBG] osdmap e69: 8 total, 8 up, 8 in 2024-07-15T06:50:46.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:45 smithi089 bash[21793]: cluster 2024-07-15T06:50:44.156822+0000 mgr.y (mgr.24272) 129 : cluster [DBG] pgmap v102: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.9 KiB/s rd, 1.3 KiB/s wr, 9 op/s 2024-07-15T06:50:46.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:45 smithi089 bash[21793]: audit 2024-07-15T06:50:44.795130+0000 mon.a (mon.0) 876 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]': finished 2024-07-15T06:50:46.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:45 smithi089 bash[21793]: audit 2024-07-15T06:50:44.795461+0000 mon.a (mon.0) 877 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]': finished 2024-07-15T06:50:46.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:45 smithi089 bash[21793]: audit 2024-07-15T06:50:44.795639+0000 mon.a (mon.0) 878 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]': finished 2024-07-15T06:50:46.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:45 smithi089 bash[21793]: cluster 2024-07-15T06:50:44.795773+0000 mon.a (mon.0) 879 : cluster [DBG] osdmap e69: 8 total, 8 up, 8 in 2024-07-15T06:50:46.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:45 smithi089 bash[31372]: cluster 2024-07-15T06:50:44.156822+0000 mgr.y (mgr.24272) 129 : cluster [DBG] pgmap v102: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.9 KiB/s rd, 1.3 KiB/s wr, 9 op/s 2024-07-15T06:50:46.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:45 smithi089 bash[31372]: audit 2024-07-15T06:50:44.795130+0000 mon.a (mon.0) 876 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]': finished 2024-07-15T06:50:46.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:45 smithi089 bash[31372]: audit 2024-07-15T06:50:44.795461+0000 mon.a (mon.0) 877 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]': finished 2024-07-15T06:50:46.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:45 smithi089 bash[31372]: audit 2024-07-15T06:50:44.795639+0000 mon.a (mon.0) 878 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]': finished 2024-07-15T06:50:46.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:45 smithi089 bash[31372]: cluster 2024-07-15T06:50:44.795773+0000 mon.a (mon.0) 879 : cluster [DBG] osdmap e69: 8 total, 8 up, 8 in 2024-07-15T06:50:47.054 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:50:46 smithi134 bash[25019]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:50:46] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-07-15T06:50:47.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:46 smithi134 bash[24068]: cluster 2024-07-15T06:50:45.792504+0000 mon.a (mon.0) 880 : cluster [DBG] osdmap e70: 8 total, 8 up, 8 in 2024-07-15T06:50:47.118 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:46 smithi089 bash[21793]: cluster 2024-07-15T06:50:45.792504+0000 mon.a (mon.0) 880 : cluster [DBG] osdmap e70: 8 total, 8 up, 8 in 2024-07-15T06:50:47.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:46 smithi089 bash[31372]: cluster 2024-07-15T06:50:45.792504+0000 mon.a (mon.0) 880 : cluster [DBG] osdmap e70: 8 total, 8 up, 8 in 2024-07-15T06:50:47.965 INFO:teuthology.run_tasks:Running task ceph.healthy... 2024-07-15T06:50:47.979 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-07-15T06:50:47.980 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-07-15T06:50:47.980 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph mgr dump --format=json 2024-07-15T06:50:48.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:47 smithi134 bash[24068]: cluster 2024-07-15T06:50:46.157425+0000 mgr.y (mgr.24272) 130 : cluster [DBG] pgmap v105: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.1 KiB/s rd, 0 B/s wr, 5 op/s 2024-07-15T06:50:48.118 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:47 smithi089 bash[21793]: cluster 2024-07-15T06:50:46.157425+0000 mgr.y (mgr.24272) 130 : cluster [DBG] pgmap v105: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.1 KiB/s rd, 0 B/s wr, 5 op/s 2024-07-15T06:50:48.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:47 smithi089 bash[31372]: cluster 2024-07-15T06:50:46.157425+0000 mgr.y (mgr.24272) 130 : cluster [DBG] pgmap v105: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.1 KiB/s rd, 0 B/s wr, 5 op/s 2024-07-15T06:50:49.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:48 smithi089 bash[21793]: cluster 2024-07-15T06:50:48.158104+0000 mgr.y (mgr.24272) 131 : cluster [DBG] pgmap v106: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-07-15T06:50:49.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:48 smithi089 bash[31372]: cluster 2024-07-15T06:50:48.158104+0000 mgr.y (mgr.24272) 131 : cluster [DBG] pgmap v106: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-07-15T06:50:49.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:48 smithi134 bash[24068]: cluster 2024-07-15T06:50:48.158104+0000 mgr.y (mgr.24272) 131 : cluster [DBG] pgmap v106: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-07-15T06:50:51.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:51 smithi134 bash[24068]: cluster 2024-07-15T06:50:50.159186+0000 mgr.y (mgr.24272) 132 : cluster [DBG] pgmap v107: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-07-15T06:50:51.618 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:51 smithi089 bash[21793]: cluster 2024-07-15T06:50:50.159186+0000 mgr.y (mgr.24272) 132 : cluster [DBG] pgmap v107: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-07-15T06:50:51.619 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:51 smithi089 bash[31372]: cluster 2024-07-15T06:50:50.159186+0000 mgr.y (mgr.24272) 132 : cluster [DBG] pgmap v107: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-07-15T06:50:52.763 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T06:50:52.915 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:50:52 smithi089 bash[22045]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:50:52] "GET /metrics HTTP/1.1" 200 239308 "" "Prometheus/2.43.0" 2024-07-15T06:50:53.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:53 smithi134 bash[24068]: cluster 2024-07-15T06:50:52.159793+0000 mgr.y (mgr.24272) 133 : cluster [DBG] pgmap v108: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-07-15T06:50:53.618 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:53 smithi089 bash[21793]: cluster 2024-07-15T06:50:52.159793+0000 mgr.y (mgr.24272) 133 : cluster [DBG] pgmap v108: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-07-15T06:50:53.619 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:53 smithi089 bash[31372]: cluster 2024-07-15T06:50:52.159793+0000 mgr.y (mgr.24272) 133 : cluster [DBG] pgmap v108: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-07-15T06:50:55.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:55 smithi134 bash[24068]: cluster 2024-07-15T06:50:54.160251+0000 mgr.y (mgr.24272) 134 : cluster [DBG] pgmap v109: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-07-15T06:50:55.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:55 smithi089 bash[21793]: cluster 2024-07-15T06:50:54.160251+0000 mgr.y (mgr.24272) 134 : cluster [DBG] pgmap v109: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-07-15T06:50:55.619 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:55 smithi089 bash[31372]: cluster 2024-07-15T06:50:54.160251+0000 mgr.y (mgr.24272) 134 : cluster [DBG] pgmap v109: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-07-15T06:50:56.315 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-07-15T06:50:57.053 INFO:teuthology.orchestra.run.smithi089.stdout:{"epoch":20,"flags":0,"active_gid":24272,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6800","nonce":2527934197},{"type":"v1","addr":"172.21.15.89:6801","nonce":2527934197}]},"active_addr":"172.21.15.89:6801/2527934197","active_change":"2024-07-15T06:47:44.086667+0000","active_mgr_features":4540138320759226367,"available":true,"standbys":[{"gid":24310,"name":"x","mgr_features":4540138320759226367,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.89:8443/","prometheus":"http://172.21.15.89:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":59,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.89:0","nonce":1034002740}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.89:0","nonce":1249162602}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.89:0","nonce":3524047244}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.89:0","nonce":2281831659}]}]} 2024-07-15T06:50:57.059 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-07-15T06:50:57.059 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-07-15T06:50:57.059 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph osd dump --format=json 2024-07-15T06:50:57.221 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:50:56 smithi134 bash[25019]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:50:56] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-07-15T06:50:57.369 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:57 smithi089 bash[21793]: cluster 2024-07-15T06:50:56.161229+0000 mgr.y (mgr.24272) 135 : cluster [DBG] pgmap v110: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:50:57.369 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:57 smithi089 bash[21793]: audit 2024-07-15T06:50:56.311787+0000 mon.a (mon.0) 881 : audit [DBG] from='client.? 172.21.15.89:0/1913109298' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-07-15T06:50:57.369 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:57 smithi089 bash[31372]: cluster 2024-07-15T06:50:56.161229+0000 mgr.y (mgr.24272) 135 : cluster [DBG] pgmap v110: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:50:57.369 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:57 smithi089 bash[31372]: audit 2024-07-15T06:50:56.311787+0000 mon.a (mon.0) 881 : audit [DBG] from='client.? 172.21.15.89:0/1913109298' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-07-15T06:50:57.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:57 smithi134 bash[24068]: cluster 2024-07-15T06:50:56.161229+0000 mgr.y (mgr.24272) 135 : cluster [DBG] pgmap v110: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:50:57.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:57 smithi134 bash[24068]: audit 2024-07-15T06:50:56.311787+0000 mon.a (mon.0) 881 : audit [DBG] from='client.? 172.21.15.89:0/1913109298' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-07-15T06:50:59.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:59 smithi134 bash[24068]: cluster 2024-07-15T06:50:58.161868+0000 mgr.y (mgr.24272) 136 : cluster [DBG] pgmap v111: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:50:59.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:50:59 smithi134 bash[24068]: audit 2024-07-15T06:50:59.192606+0000 mon.a (mon.0) 882 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:50:59.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:59 smithi089 bash[21793]: cluster 2024-07-15T06:50:58.161868+0000 mgr.y (mgr.24272) 136 : cluster [DBG] pgmap v111: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:50:59.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:50:59 smithi089 bash[21793]: audit 2024-07-15T06:50:59.192606+0000 mon.a (mon.0) 882 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:50:59.619 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:59 smithi089 bash[31372]: cluster 2024-07-15T06:50:58.161868+0000 mgr.y (mgr.24272) 136 : cluster [DBG] pgmap v111: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:50:59.619 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:50:59 smithi089 bash[31372]: audit 2024-07-15T06:50:59.192606+0000 mon.a (mon.0) 882 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:51:01.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:51:01 smithi134 bash[24068]: cluster 2024-07-15T06:51:00.162912+0000 mgr.y (mgr.24272) 137 : cluster [DBG] pgmap v112: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:51:01.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:51:01 smithi089 bash[21793]: cluster 2024-07-15T06:51:00.162912+0000 mgr.y (mgr.24272) 137 : cluster [DBG] pgmap v112: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:51:01.619 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:51:01 smithi089 bash[31372]: cluster 2024-07-15T06:51:00.162912+0000 mgr.y (mgr.24272) 137 : cluster [DBG] pgmap v112: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:51:01.839 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T06:51:03.002 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:51:02 smithi089 bash[22045]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:51:02] "GET /metrics HTTP/1.1" 200 239289 "" "Prometheus/2.43.0" 2024-07-15T06:51:03.286 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:51:03 smithi089 bash[31372]: cluster 2024-07-15T06:51:02.163565+0000 mgr.y (mgr.24272) 138 : cluster [DBG] pgmap v113: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:51:03.287 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:51:03 smithi089 bash[21793]: cluster 2024-07-15T06:51:02.163565+0000 mgr.y (mgr.24272) 138 : cluster [DBG] pgmap v113: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:51:03.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:51:03 smithi134 bash[24068]: cluster 2024-07-15T06:51:02.163565+0000 mgr.y (mgr.24272) 138 : cluster [DBG] pgmap v113: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:51:05.209 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-07-15T06:51:05.209 INFO:teuthology.orchestra.run.smithi089.stdout:{"epoch":70,"fsid":"75ba02bc-4274-11ef-bcad-c7b262605968","created":"2024-07-15T06:37:09.634877+0000","modified":"2024-07-15T06:50:45.785629+0000","last_up_change":"2024-07-15T06:46:45.306680+0000","last_in_change":"2024-07-15T06:46:19.019927+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-07-15T06:42:21.444347+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2024-07-15T06:48:27.276344+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-07-15T06:50:28.088431+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"66","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-07-15T06:50:28.643335+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"67","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"79357114-251f-489d-bef5-f6ec37002212","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6802","nonce":843843653},{"type":"v1","addr":"172.21.15.89:6803","nonce":843843653}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6804","nonce":843843653},{"type":"v1","addr":"172.21.15.89:6805","nonce":843843653}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6808","nonce":843843653},{"type":"v1","addr":"172.21.15.89:6809","nonce":843843653}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6806","nonce":843843653},{"type":"v1","addr":"172.21.15.89:6807","nonce":843843653}]},"public_addr":"172.21.15.89:6803/843843653","cluster_addr":"172.21.15.89:6805/843843653","heartbeat_back_addr":"172.21.15.89:6809/843843653","heartbeat_front_addr":"172.21.15.89:6807/843843653","state":["exists","up"]},{"osd":1,"uuid":"d304859e-317a-414e-a16c-f097fc4b2702","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6810","nonce":1536075263},{"type":"v1","addr":"172.21.15.89:6811","nonce":1536075263}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6812","nonce":1536075263},{"type":"v1","addr":"172.21.15.89:6813","nonce":1536075263}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6816","nonce":1536075263},{"type":"v1","addr":"172.21.15.89:6817","nonce":1536075263}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6814","nonce":1536075263},{"type":"v1","addr":"172.21.15.89:6815","nonce":1536075263}]},"public_addr":"172.21.15.89:6811/1536075263","cluster_addr":"172.21.15.89:6813/1536075263","heartbeat_back_addr":"172.21.15.89:6817/1536075263","heartbeat_front_addr":"172.21.15.89:6815/1536075263","state":["exists","up"]},{"osd":2,"uuid":"60cde686-83bb-4ca9-a0f1-6668710e16f1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":69,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6818","nonce":3236432318},{"type":"v1","addr":"172.21.15.89:6819","nonce":3236432318}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6820","nonce":3236432318},{"type":"v1","addr":"172.21.15.89:6821","nonce":3236432318}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6824","nonce":3236432318},{"type":"v1","addr":"172.21.15.89:6825","nonce":3236432318}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6822","nonce":3236432318},{"type":"v1","addr":"172.21.15.89:6823","nonce":3236432318}]},"public_addr":"172.21.15.89:6819/3236432318","cluster_addr":"172.21.15.89:6821/3236432318","heartbeat_back_addr":"172.21.15.89:6825/3236432318","heartbeat_front_addr":"172.21.15.89:6823/3236432318","state":["exists","up"]},{"osd":3,"uuid":"8af52bbd-4bc9-4b13-b525-26c92e69c29f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6826","nonce":4083663609},{"type":"v1","addr":"172.21.15.89:6827","nonce":4083663609}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6828","nonce":4083663609},{"type":"v1","addr":"172.21.15.89:6829","nonce":4083663609}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6832","nonce":4083663609},{"type":"v1","addr":"172.21.15.89:6833","nonce":4083663609}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6830","nonce":4083663609},{"type":"v1","addr":"172.21.15.89:6831","nonce":4083663609}]},"public_addr":"172.21.15.89:6827/4083663609","cluster_addr":"172.21.15.89:6829/4083663609","heartbeat_back_addr":"172.21.15.89:6833/4083663609","heartbeat_front_addr":"172.21.15.89:6831/4083663609","state":["exists","up"]},{"osd":4,"uuid":"b97874a6-e72a-4068-9c71-a50b8790be2c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":69,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6800","nonce":3588917663},{"type":"v1","addr":"172.21.15.134:6801","nonce":3588917663}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6802","nonce":3588917663},{"type":"v1","addr":"172.21.15.134:6803","nonce":3588917663}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6806","nonce":3588917663},{"type":"v1","addr":"172.21.15.134:6807","nonce":3588917663}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6804","nonce":3588917663},{"type":"v1","addr":"172.21.15.134:6805","nonce":3588917663}]},"public_addr":"172.21.15.134:6801/3588917663","cluster_addr":"172.21.15.134:6803/3588917663","heartbeat_back_addr":"172.21.15.134:6807/3588917663","heartbeat_front_addr":"172.21.15.134:6805/3588917663","state":["exists","up"]},{"osd":5,"uuid":"a834c8ee-e034-423d-ab67-1ab33bc15a97","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6808","nonce":823577783},{"type":"v1","addr":"172.21.15.134:6809","nonce":823577783}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6810","nonce":823577783},{"type":"v1","addr":"172.21.15.134:6811","nonce":823577783}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6814","nonce":823577783},{"type":"v1","addr":"172.21.15.134:6815","nonce":823577783}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6812","nonce":823577783},{"type":"v1","addr":"172.21.15.134:6813","nonce":823577783}]},"public_addr":"172.21.15.134:6809/823577783","cluster_addr":"172.21.15.134:6811/823577783","heartbeat_back_addr":"172.21.15.134:6815/823577783","heartbeat_front_addr":"172.21.15.134:6813/823577783","state":["exists","up"]},{"osd":6,"uuid":"446cb8fa-8ddb-4f7a-a8a4-c7281e950f74","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":69,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6816","nonce":4010812142},{"type":"v1","addr":"172.21.15.134:6817","nonce":4010812142}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6818","nonce":4010812142},{"type":"v1","addr":"172.21.15.134:6819","nonce":4010812142}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6822","nonce":4010812142},{"type":"v1","addr":"172.21.15.134:6823","nonce":4010812142}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6820","nonce":4010812142},{"type":"v1","addr":"172.21.15.134:6821","nonce":4010812142}]},"public_addr":"172.21.15.134:6817/4010812142","cluster_addr":"172.21.15.134:6819/4010812142","heartbeat_back_addr":"172.21.15.134:6823/4010812142","heartbeat_front_addr":"172.21.15.134:6821/4010812142","state":["exists","up"]},{"osd":7,"uuid":"87f3a30f-2935-4b92-af37-bd866696838f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6824","nonce":3883480074},{"type":"v1","addr":"172.21.15.134:6825","nonce":3883480074}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6826","nonce":3883480074},{"type":"v1","addr":"172.21.15.134:6827","nonce":3883480074}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6830","nonce":3883480074},{"type":"v1","addr":"172.21.15.134:6831","nonce":3883480074}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6828","nonce":3883480074},{"type":"v1","addr":"172.21.15.134:6829","nonce":3883480074}]},"public_addr":"172.21.15.134:6825/3883480074","cluster_addr":"172.21.15.134:6827/3883480074","heartbeat_back_addr":"172.21.15.134:6831/3883480074","heartbeat_front_addr":"172.21.15.134:6829/3883480074","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-15T06:40:36.567653+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-15T06:41:26.849575+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-15T06:42:18.731018+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-15T06:43:11.604111+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-15T06:44:03.847505+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-15T06:44:56.895305+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-15T06:45:50.039730+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-15T06:46:43.462786+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.a","mappings":[{"from":1,"to":2}]},{"pgid":"4.13","mappings":[{"from":1,"to":2}]},{"pgid":"4.1c","mappings":[{"from":1,"to":5}]}],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.89:6835/4041986456":"2024-07-16T06:50:36.703871+0000","172.21.15.89:6834/4041986456":"2024-07-16T06:50:36.703871+0000","172.21.15.89:6801/265242087":"2024-07-16T06:47:44.086461+0000","172.21.15.89:0/1100221668":"2024-07-16T06:47:44.086461+0000","172.21.15.89:0/1877203535":"2024-07-16T06:37:35.981306+0000","172.21.15.89:0/1750227302":"2024-07-16T06:38:06.841135+0000","172.21.15.89:6800/2189595088":"2024-07-16T06:37:35.981306+0000","172.21.15.89:6801/2189595088":"2024-07-16T06:37:35.981306+0000","172.21.15.89:0/3465936199":"2024-07-16T06:47:44.086461+0000","172.21.15.89:0/1228522973":"2024-07-16T06:37:35.981306+0000","172.21.15.89:0/889381338":"2024-07-16T06:38:06.841135+0000","172.21.15.89:0/1672418864":"2024-07-16T06:38:06.841135+0000","172.21.15.89:0/2127823469":"2024-07-16T06:37:35.981306+0000","172.21.15.89:6801/2919301358":"2024-07-16T06:38:06.841135+0000","172.21.15.89:6800/265242087":"2024-07-16T06:47:44.086461+0000","172.21.15.89:6800/2919301358":"2024-07-16T06:38:06.841135+0000","172.21.15.89:0/3680199495":"2024-07-16T06:47:44.086461+0000","172.21.15.89:0/1054475873":"2024-07-16T06:47:44.086461+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-07-15T06:51:05.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:51:05 smithi134 bash[24068]: cluster 2024-07-15T06:51:04.164205+0000 mgr.y (mgr.24272) 139 : cluster [DBG] pgmap v114: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:51:05.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:51:05 smithi134 bash[24068]: audit 2024-07-15T06:51:05.213638+0000 mon.a (mon.0) 883 : audit [DBG] from='client.? 172.21.15.89:0/1776506456' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-07-15T06:51:05.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:51:05 smithi089 bash[21793]: cluster 2024-07-15T06:51:04.164205+0000 mgr.y (mgr.24272) 139 : cluster [DBG] pgmap v114: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:51:05.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:51:05 smithi089 bash[21793]: audit 2024-07-15T06:51:05.213638+0000 mon.a (mon.0) 883 : audit [DBG] from='client.? 172.21.15.89:0/1776506456' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-07-15T06:51:05.619 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:51:05 smithi089 bash[31372]: cluster 2024-07-15T06:51:04.164205+0000 mgr.y (mgr.24272) 139 : cluster [DBG] pgmap v114: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:51:05.619 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:51:05 smithi089 bash[31372]: audit 2024-07-15T06:51:05.213638+0000 mon.a (mon.0) 883 : audit [DBG] from='client.? 172.21.15.89:0/1776506456' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-07-15T06:51:05.914 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-07-15T06:51:05.914 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph osd dump --format=json 2024-07-15T06:51:07.242 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:51:06 smithi134 bash[25019]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:51:06] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-07-15T06:51:07.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:51:07 smithi134 bash[24068]: cluster 2024-07-15T06:51:06.165264+0000 mgr.y (mgr.24272) 140 : cluster [DBG] pgmap v115: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:51:07.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:51:07 smithi089 bash[21793]: cluster 2024-07-15T06:51:06.165264+0000 mgr.y (mgr.24272) 140 : cluster [DBG] pgmap v115: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:51:07.619 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:51:07 smithi089 bash[31372]: cluster 2024-07-15T06:51:06.165264+0000 mgr.y (mgr.24272) 140 : cluster [DBG] pgmap v115: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:51:09.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:51:09 smithi134 bash[24068]: cluster 2024-07-15T06:51:08.165899+0000 mgr.y (mgr.24272) 141 : cluster [DBG] pgmap v116: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:51:09.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:51:09 smithi089 bash[21793]: cluster 2024-07-15T06:51:08.165899+0000 mgr.y (mgr.24272) 141 : cluster [DBG] pgmap v116: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:51:09.619 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:51:09 smithi089 bash[31372]: cluster 2024-07-15T06:51:08.165899+0000 mgr.y (mgr.24272) 141 : cluster [DBG] pgmap v116: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:51:10.690 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T06:51:11.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:51:11 smithi134 bash[24068]: cluster 2024-07-15T06:51:10.166950+0000 mgr.y (mgr.24272) 142 : cluster [DBG] pgmap v117: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:51:11.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:51:11 smithi089 bash[21793]: cluster 2024-07-15T06:51:10.166950+0000 mgr.y (mgr.24272) 142 : cluster [DBG] pgmap v117: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:51:11.619 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:51:11 smithi089 bash[31372]: cluster 2024-07-15T06:51:10.166950+0000 mgr.y (mgr.24272) 142 : cluster [DBG] pgmap v117: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:51:12.869 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:51:12 smithi089 bash[22045]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:51:12] "GET /metrics HTTP/1.1" 200 239289 "" "Prometheus/2.43.0" 2024-07-15T06:51:13.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:51:13 smithi134 bash[24068]: cluster 2024-07-15T06:51:12.167460+0000 mgr.y (mgr.24272) 143 : cluster [DBG] pgmap v118: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:51:13.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:51:13 smithi089 bash[21793]: cluster 2024-07-15T06:51:12.167460+0000 mgr.y (mgr.24272) 143 : cluster [DBG] pgmap v118: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:51:13.619 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:51:13 smithi089 bash[31372]: cluster 2024-07-15T06:51:12.167460+0000 mgr.y (mgr.24272) 143 : cluster [DBG] pgmap v118: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:51:14.369 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:51:14 smithi089 bash[21793]: audit 2024-07-15T06:51:14.192536+0000 mon.a (mon.0) 884 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:51:14.369 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:51:14 smithi089 bash[31372]: audit 2024-07-15T06:51:14.192536+0000 mon.a (mon.0) 884 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:51:14.443 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-07-15T06:51:14.444 INFO:teuthology.orchestra.run.smithi089.stdout:{"epoch":70,"fsid":"75ba02bc-4274-11ef-bcad-c7b262605968","created":"2024-07-15T06:37:09.634877+0000","modified":"2024-07-15T06:50:45.785629+0000","last_up_change":"2024-07-15T06:46:45.306680+0000","last_in_change":"2024-07-15T06:46:19.019927+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-07-15T06:42:21.444347+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2024-07-15T06:48:27.276344+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-07-15T06:50:28.088431+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"66","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-07-15T06:50:28.643335+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"67","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"79357114-251f-489d-bef5-f6ec37002212","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6802","nonce":843843653},{"type":"v1","addr":"172.21.15.89:6803","nonce":843843653}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6804","nonce":843843653},{"type":"v1","addr":"172.21.15.89:6805","nonce":843843653}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6808","nonce":843843653},{"type":"v1","addr":"172.21.15.89:6809","nonce":843843653}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6806","nonce":843843653},{"type":"v1","addr":"172.21.15.89:6807","nonce":843843653}]},"public_addr":"172.21.15.89:6803/843843653","cluster_addr":"172.21.15.89:6805/843843653","heartbeat_back_addr":"172.21.15.89:6809/843843653","heartbeat_front_addr":"172.21.15.89:6807/843843653","state":["exists","up"]},{"osd":1,"uuid":"d304859e-317a-414e-a16c-f097fc4b2702","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6810","nonce":1536075263},{"type":"v1","addr":"172.21.15.89:6811","nonce":1536075263}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6812","nonce":1536075263},{"type":"v1","addr":"172.21.15.89:6813","nonce":1536075263}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6816","nonce":1536075263},{"type":"v1","addr":"172.21.15.89:6817","nonce":1536075263}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6814","nonce":1536075263},{"type":"v1","addr":"172.21.15.89:6815","nonce":1536075263}]},"public_addr":"172.21.15.89:6811/1536075263","cluster_addr":"172.21.15.89:6813/1536075263","heartbeat_back_addr":"172.21.15.89:6817/1536075263","heartbeat_front_addr":"172.21.15.89:6815/1536075263","state":["exists","up"]},{"osd":2,"uuid":"60cde686-83bb-4ca9-a0f1-6668710e16f1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":69,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6818","nonce":3236432318},{"type":"v1","addr":"172.21.15.89:6819","nonce":3236432318}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6820","nonce":3236432318},{"type":"v1","addr":"172.21.15.89:6821","nonce":3236432318}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6824","nonce":3236432318},{"type":"v1","addr":"172.21.15.89:6825","nonce":3236432318}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6822","nonce":3236432318},{"type":"v1","addr":"172.21.15.89:6823","nonce":3236432318}]},"public_addr":"172.21.15.89:6819/3236432318","cluster_addr":"172.21.15.89:6821/3236432318","heartbeat_back_addr":"172.21.15.89:6825/3236432318","heartbeat_front_addr":"172.21.15.89:6823/3236432318","state":["exists","up"]},{"osd":3,"uuid":"8af52bbd-4bc9-4b13-b525-26c92e69c29f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6826","nonce":4083663609},{"type":"v1","addr":"172.21.15.89:6827","nonce":4083663609}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6828","nonce":4083663609},{"type":"v1","addr":"172.21.15.89:6829","nonce":4083663609}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6832","nonce":4083663609},{"type":"v1","addr":"172.21.15.89:6833","nonce":4083663609}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6830","nonce":4083663609},{"type":"v1","addr":"172.21.15.89:6831","nonce":4083663609}]},"public_addr":"172.21.15.89:6827/4083663609","cluster_addr":"172.21.15.89:6829/4083663609","heartbeat_back_addr":"172.21.15.89:6833/4083663609","heartbeat_front_addr":"172.21.15.89:6831/4083663609","state":["exists","up"]},{"osd":4,"uuid":"b97874a6-e72a-4068-9c71-a50b8790be2c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":69,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6800","nonce":3588917663},{"type":"v1","addr":"172.21.15.134:6801","nonce":3588917663}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6802","nonce":3588917663},{"type":"v1","addr":"172.21.15.134:6803","nonce":3588917663}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6806","nonce":3588917663},{"type":"v1","addr":"172.21.15.134:6807","nonce":3588917663}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6804","nonce":3588917663},{"type":"v1","addr":"172.21.15.134:6805","nonce":3588917663}]},"public_addr":"172.21.15.134:6801/3588917663","cluster_addr":"172.21.15.134:6803/3588917663","heartbeat_back_addr":"172.21.15.134:6807/3588917663","heartbeat_front_addr":"172.21.15.134:6805/3588917663","state":["exists","up"]},{"osd":5,"uuid":"a834c8ee-e034-423d-ab67-1ab33bc15a97","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6808","nonce":823577783},{"type":"v1","addr":"172.21.15.134:6809","nonce":823577783}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6810","nonce":823577783},{"type":"v1","addr":"172.21.15.134:6811","nonce":823577783}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6814","nonce":823577783},{"type":"v1","addr":"172.21.15.134:6815","nonce":823577783}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6812","nonce":823577783},{"type":"v1","addr":"172.21.15.134:6813","nonce":823577783}]},"public_addr":"172.21.15.134:6809/823577783","cluster_addr":"172.21.15.134:6811/823577783","heartbeat_back_addr":"172.21.15.134:6815/823577783","heartbeat_front_addr":"172.21.15.134:6813/823577783","state":["exists","up"]},{"osd":6,"uuid":"446cb8fa-8ddb-4f7a-a8a4-c7281e950f74","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":69,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6816","nonce":4010812142},{"type":"v1","addr":"172.21.15.134:6817","nonce":4010812142}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6818","nonce":4010812142},{"type":"v1","addr":"172.21.15.134:6819","nonce":4010812142}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6822","nonce":4010812142},{"type":"v1","addr":"172.21.15.134:6823","nonce":4010812142}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6820","nonce":4010812142},{"type":"v1","addr":"172.21.15.134:6821","nonce":4010812142}]},"public_addr":"172.21.15.134:6817/4010812142","cluster_addr":"172.21.15.134:6819/4010812142","heartbeat_back_addr":"172.21.15.134:6823/4010812142","heartbeat_front_addr":"172.21.15.134:6821/4010812142","state":["exists","up"]},{"osd":7,"uuid":"87f3a30f-2935-4b92-af37-bd866696838f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6824","nonce":3883480074},{"type":"v1","addr":"172.21.15.134:6825","nonce":3883480074}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6826","nonce":3883480074},{"type":"v1","addr":"172.21.15.134:6827","nonce":3883480074}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6830","nonce":3883480074},{"type":"v1","addr":"172.21.15.134:6831","nonce":3883480074}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6828","nonce":3883480074},{"type":"v1","addr":"172.21.15.134:6829","nonce":3883480074}]},"public_addr":"172.21.15.134:6825/3883480074","cluster_addr":"172.21.15.134:6827/3883480074","heartbeat_back_addr":"172.21.15.134:6831/3883480074","heartbeat_front_addr":"172.21.15.134:6829/3883480074","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-15T06:40:36.567653+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-15T06:41:26.849575+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-15T06:42:18.731018+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-15T06:43:11.604111+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-15T06:44:03.847505+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-15T06:44:56.895305+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-15T06:45:50.039730+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-15T06:46:43.462786+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.a","mappings":[{"from":1,"to":2}]},{"pgid":"4.13","mappings":[{"from":1,"to":2}]},{"pgid":"4.1c","mappings":[{"from":1,"to":5}]}],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.89:6835/4041986456":"2024-07-16T06:50:36.703871+0000","172.21.15.89:6834/4041986456":"2024-07-16T06:50:36.703871+0000","172.21.15.89:6801/265242087":"2024-07-16T06:47:44.086461+0000","172.21.15.89:0/1100221668":"2024-07-16T06:47:44.086461+0000","172.21.15.89:0/1877203535":"2024-07-16T06:37:35.981306+0000","172.21.15.89:0/1750227302":"2024-07-16T06:38:06.841135+0000","172.21.15.89:6800/2189595088":"2024-07-16T06:37:35.981306+0000","172.21.15.89:6801/2189595088":"2024-07-16T06:37:35.981306+0000","172.21.15.89:0/3465936199":"2024-07-16T06:47:44.086461+0000","172.21.15.89:0/1228522973":"2024-07-16T06:37:35.981306+0000","172.21.15.89:0/889381338":"2024-07-16T06:38:06.841135+0000","172.21.15.89:0/1672418864":"2024-07-16T06:38:06.841135+0000","172.21.15.89:0/2127823469":"2024-07-16T06:37:35.981306+0000","172.21.15.89:6801/2919301358":"2024-07-16T06:38:06.841135+0000","172.21.15.89:6800/265242087":"2024-07-16T06:47:44.086461+0000","172.21.15.89:6800/2919301358":"2024-07-16T06:38:06.841135+0000","172.21.15.89:0/3680199495":"2024-07-16T06:47:44.086461+0000","172.21.15.89:0/1054475873":"2024-07-16T06:47:44.086461+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-07-15T06:51:14.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:51:14 smithi134 bash[24068]: audit 2024-07-15T06:51:14.192536+0000 mon.a (mon.0) 884 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:51:15.127 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-07-15T06:51:15.127 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-07-15T06:51:15.128 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-07-15T06:51:15.128 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-07-15T06:51:15.128 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-07-15T06:51:15.129 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-07-15T06:51:15.129 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-07-15T06:51:15.129 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-07-15T06:51:15.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:51:15 smithi134 bash[24068]: cluster 2024-07-15T06:51:14.167947+0000 mgr.y (mgr.24272) 144 : cluster [DBG] pgmap v119: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:51:15.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:51:15 smithi134 bash[24068]: audit 2024-07-15T06:51:14.447761+0000 mon.c (mon.2) 25 : audit [DBG] from='client.? 172.21.15.89:0/1944099322' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-07-15T06:51:15.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:51:15 smithi089 bash[21793]: cluster 2024-07-15T06:51:14.167947+0000 mgr.y (mgr.24272) 144 : cluster [DBG] pgmap v119: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:51:15.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:51:15 smithi089 bash[21793]: audit 2024-07-15T06:51:14.447761+0000 mon.c (mon.2) 25 : audit [DBG] from='client.? 172.21.15.89:0/1944099322' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-07-15T06:51:15.619 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:51:15 smithi089 bash[31372]: cluster 2024-07-15T06:51:14.167947+0000 mgr.y (mgr.24272) 144 : cluster [DBG] pgmap v119: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:51:15.619 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:51:15 smithi089 bash[31372]: audit 2024-07-15T06:51:14.447761+0000 mon.c (mon.2) 25 : audit [DBG] from='client.? 172.21.15.89:0/1944099322' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-07-15T06:51:17.277 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:51:16 smithi134 bash[25019]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:51:16] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-07-15T06:51:17.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:51:17 smithi134 bash[24068]: cluster 2024-07-15T06:51:16.169004+0000 mgr.y (mgr.24272) 145 : cluster [DBG] pgmap v120: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:51:17.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:51:17 smithi089 bash[21793]: cluster 2024-07-15T06:51:16.169004+0000 mgr.y (mgr.24272) 145 : cluster [DBG] pgmap v120: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:51:17.619 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:51:17 smithi089 bash[31372]: cluster 2024-07-15T06:51:16.169004+0000 mgr.y (mgr.24272) 145 : cluster [DBG] pgmap v120: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:51:19.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:51:19 smithi134 bash[24068]: cluster 2024-07-15T06:51:18.169606+0000 mgr.y (mgr.24272) 146 : cluster [DBG] pgmap v121: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:51:19.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:51:19 smithi089 bash[21793]: cluster 2024-07-15T06:51:18.169606+0000 mgr.y (mgr.24272) 146 : cluster [DBG] pgmap v121: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:51:19.619 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:51:19 smithi089 bash[31372]: cluster 2024-07-15T06:51:18.169606+0000 mgr.y (mgr.24272) 146 : cluster [DBG] pgmap v121: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:51:20.041 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T06:51:20.041 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T06:51:20.042 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T06:51:20.042 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T06:51:20.045 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T06:51:20.045 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T06:51:20.046 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T06:51:20.050 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T06:51:21.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:51:21 smithi134 bash[24068]: cluster 2024-07-15T06:51:20.170573+0000 mgr.y (mgr.24272) 147 : cluster [DBG] pgmap v122: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:51:21.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:51:21 smithi089 bash[21793]: cluster 2024-07-15T06:51:20.170573+0000 mgr.y (mgr.24272) 147 : cluster [DBG] pgmap v122: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:51:21.619 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:51:21 smithi089 bash[31372]: cluster 2024-07-15T06:51:20.170573+0000 mgr.y (mgr.24272) 147 : cluster [DBG] pgmap v122: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:51:22.934 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:51:22 smithi089 bash[22045]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:51:22] "GET /metrics HTTP/1.1" 200 239269 "" "Prometheus/2.43.0" 2024-07-15T06:51:23.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:51:23 smithi134 bash[24068]: cluster 2024-07-15T06:51:22.171245+0000 mgr.y (mgr.24272) 148 : cluster [DBG] pgmap v123: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:51:23.623 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:51:23 smithi089 bash[31372]: cluster 2024-07-15T06:51:22.171245+0000 mgr.y (mgr.24272) 148 : cluster [DBG] pgmap v123: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:51:23.623 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:51:23 smithi089 bash[21793]: cluster 2024-07-15T06:51:22.171245+0000 mgr.y (mgr.24272) 148 : cluster [DBG] pgmap v123: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:51:25.369 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:51:25 smithi089 bash[21793]: cluster 2024-07-15T06:51:24.171790+0000 mgr.y (mgr.24272) 149 : cluster [DBG] pgmap v124: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:51:25.369 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:51:25 smithi089 bash[31372]: cluster 2024-07-15T06:51:24.171790+0000 mgr.y (mgr.24272) 149 : cluster [DBG] pgmap v124: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:51:25.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:51:25 smithi134 bash[24068]: cluster 2024-07-15T06:51:24.171790+0000 mgr.y (mgr.24272) 149 : cluster [DBG] pgmap v124: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:51:27.296 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:51:26 smithi134 bash[25019]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:51:26] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-07-15T06:51:27.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:51:27 smithi134 bash[24068]: cluster 2024-07-15T06:51:26.172889+0000 mgr.y (mgr.24272) 150 : cluster [DBG] pgmap v125: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:51:27.581 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:51:27 smithi089 bash[31372]: cluster 2024-07-15T06:51:26.172889+0000 mgr.y (mgr.24272) 150 : cluster [DBG] pgmap v125: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:51:27.582 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:51:27 smithi089 bash[21793]: cluster 2024-07-15T06:51:26.172889+0000 mgr.y (mgr.24272) 150 : cluster [DBG] pgmap v125: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:51:29.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:51:29 smithi134 bash[24068]: cluster 2024-07-15T06:51:28.173507+0000 mgr.y (mgr.24272) 151 : cluster [DBG] pgmap v126: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:51:29.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:51:29 smithi134 bash[24068]: audit 2024-07-15T06:51:29.193827+0000 mon.a (mon.0) 885 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:51:29.705 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:51:29 smithi089 bash[21793]: cluster 2024-07-15T06:51:28.173507+0000 mgr.y (mgr.24272) 151 : cluster [DBG] pgmap v126: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:51:29.705 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:51:29 smithi089 bash[21793]: audit 2024-07-15T06:51:29.193827+0000 mon.a (mon.0) 885 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:51:29.705 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:51:29 smithi089 bash[31372]: cluster 2024-07-15T06:51:28.173507+0000 mgr.y (mgr.24272) 151 : cluster [DBG] pgmap v126: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:51:29.705 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:51:29 smithi089 bash[31372]: audit 2024-07-15T06:51:29.193827+0000 mon.a (mon.0) 885 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:51:31.536 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:51:31 smithi089 bash[21793]: cluster 2024-07-15T06:51:30.174494+0000 mgr.y (mgr.24272) 152 : cluster [DBG] pgmap v127: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:51:31.536 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:51:31 smithi089 bash[31372]: cluster 2024-07-15T06:51:30.174494+0000 mgr.y (mgr.24272) 152 : cluster [DBG] pgmap v127: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:51:31.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:51:31 smithi134 bash[24068]: cluster 2024-07-15T06:51:30.174494+0000 mgr.y (mgr.24272) 152 : cluster [DBG] pgmap v127: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:51:32.892 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:51:32 smithi089 bash[22045]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:51:32] "GET /metrics HTTP/1.1" 200 239257 "" "Prometheus/2.43.0" 2024-07-15T06:51:33.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:51:33 smithi089 bash[21793]: cluster 2024-07-15T06:51:32.175113+0000 mgr.y (mgr.24272) 153 : cluster [DBG] pgmap v128: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:51:33.619 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:51:33 smithi089 bash[31372]: cluster 2024-07-15T06:51:32.175113+0000 mgr.y (mgr.24272) 153 : cluster [DBG] pgmap v128: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:51:33.765 INFO:teuthology.orchestra.run.smithi089.stdout:107374182503 2024-07-15T06:51:33.765 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-07-15T06:51:33.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:51:33 smithi134 bash[24068]: cluster 2024-07-15T06:51:32.175113+0000 mgr.y (mgr.24272) 153 : cluster [DBG] pgmap v128: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:51:35.496 INFO:teuthology.orchestra.run.smithi089.stdout:163208757329 2024-07-15T06:51:35.496 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-07-15T06:51:35.507 INFO:teuthology.orchestra.run.smithi089.stdout:34359738502 2024-07-15T06:51:35.507 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-07-15T06:51:35.567 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:51:35 smithi089 bash[21793]: cluster 2024-07-15T06:51:34.175875+0000 mgr.y (mgr.24272) 154 : cluster [DBG] pgmap v129: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:51:35.567 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:51:35 smithi089 bash[31372]: cluster 2024-07-15T06:51:34.175875+0000 mgr.y (mgr.24272) 154 : cluster [DBG] pgmap v129: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:51:35.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:51:35 smithi134 bash[24068]: cluster 2024-07-15T06:51:34.175875+0000 mgr.y (mgr.24272) 154 : cluster [DBG] pgmap v129: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:51:36.806 INFO:teuthology.orchestra.run.smithi089.stdout:231928234045 2024-07-15T06:51:36.806 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-07-15T06:51:37.304 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:51:36 smithi134 bash[25019]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:51:36] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-07-15T06:51:37.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:51:37 smithi089 bash[21793]: cluster 2024-07-15T06:51:36.176533+0000 mgr.y (mgr.24272) 155 : cluster [DBG] pgmap v130: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:51:37.619 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:51:37 smithi089 bash[31372]: cluster 2024-07-15T06:51:36.176533+0000 mgr.y (mgr.24272) 155 : cluster [DBG] pgmap v130: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:51:37.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:51:37 smithi134 bash[24068]: cluster 2024-07-15T06:51:36.176533+0000 mgr.y (mgr.24272) 155 : cluster [DBG] pgmap v130: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:51:37.863 INFO:teuthology.orchestra.run.smithi089.stdout:77309411440 2024-07-15T06:51:37.863 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-07-15T06:51:38.147 INFO:teuthology.orchestra.run.smithi089.stdout:128849018973 2024-07-15T06:51:38.147 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-07-15T06:51:39.225 INFO:teuthology.orchestra.run.smithi089.stdout:197568495688 2024-07-15T06:51:39.225 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-07-15T06:51:39.238 INFO:teuthology.orchestra.run.smithi089.stdout:55834574972 2024-07-15T06:51:39.239 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-07-15T06:51:39.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:51:39 smithi089 bash[21793]: cluster 2024-07-15T06:51:38.177355+0000 mgr.y (mgr.24272) 156 : cluster [DBG] pgmap v131: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:51:39.619 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:51:39 smithi089 bash[31372]: cluster 2024-07-15T06:51:38.177355+0000 mgr.y (mgr.24272) 156 : cluster [DBG] pgmap v131: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:51:39.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:51:39 smithi134 bash[24068]: cluster 2024-07-15T06:51:38.177355+0000 mgr.y (mgr.24272) 156 : cluster [DBG] pgmap v131: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:51:41.597 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T06:51:41.597 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T06:51:41.598 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T06:51:41.600 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T06:51:41.618 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:51:41 smithi089 bash[31372]: cluster 2024-07-15T06:51:40.178416+0000 mgr.y (mgr.24272) 157 : cluster [DBG] pgmap v132: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:51:41.619 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:51:41 smithi089 bash[31372]: audit 2024-07-15T06:51:41.209829+0000 mon.a (mon.0) 886 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:51:41.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:51:41 smithi089 bash[21793]: cluster 2024-07-15T06:51:40.178416+0000 mgr.y (mgr.24272) 157 : cluster [DBG] pgmap v132: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:51:41.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:51:41 smithi089 bash[21793]: audit 2024-07-15T06:51:41.209829+0000 mon.a (mon.0) 886 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:51:41.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:51:41 smithi134 bash[24068]: cluster 2024-07-15T06:51:40.178416+0000 mgr.y (mgr.24272) 157 : cluster [DBG] pgmap v132: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:51:41.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:51:41 smithi134 bash[24068]: audit 2024-07-15T06:51:41.209829+0000 mon.a (mon.0) 886 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:51:42.617 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T06:51:42.618 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T06:51:42.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:51:42 smithi089 bash[21793]: audit 2024-07-15T06:51:41.696123+0000 mon.a (mon.0) 887 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:51:42.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:51:42 smithi089 bash[21793]: audit 2024-07-15T06:51:41.696914+0000 mon.a (mon.0) 888 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:51:42.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:51:42 smithi089 bash[21793]: audit 2024-07-15T06:51:41.703953+0000 mon.a (mon.0) 889 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:51:42.619 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:51:42 smithi089 bash[31372]: audit 2024-07-15T06:51:41.696123+0000 mon.a (mon.0) 887 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:51:42.619 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:51:42 smithi089 bash[31372]: audit 2024-07-15T06:51:41.696914+0000 mon.a (mon.0) 888 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:51:42.619 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:51:42 smithi089 bash[31372]: audit 2024-07-15T06:51:41.703953+0000 mon.a (mon.0) 889 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:51:42.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:51:42 smithi134 bash[24068]: audit 2024-07-15T06:51:41.696123+0000 mon.a (mon.0) 887 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:51:42.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:51:42 smithi134 bash[24068]: audit 2024-07-15T06:51:41.696914+0000 mon.a (mon.0) 888 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:51:42.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:51:42 smithi134 bash[24068]: audit 2024-07-15T06:51:41.703953+0000 mon.a (mon.0) 889 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:51:42.970 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:51:42 smithi089 bash[22045]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:51:42] "GET /metrics HTTP/1.1" 200 239257 "" "Prometheus/2.43.0" 2024-07-15T06:51:43.639 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T06:51:43.639 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T06:51:44.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:51:43 smithi134 bash[24068]: cluster 2024-07-15T06:51:42.178956+0000 mgr.y (mgr.24272) 158 : cluster [DBG] pgmap v133: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:51:44.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:51:43 smithi089 bash[21793]: cluster 2024-07-15T06:51:42.178956+0000 mgr.y (mgr.24272) 158 : cluster [DBG] pgmap v133: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:51:44.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:51:43 smithi089 bash[31372]: cluster 2024-07-15T06:51:42.178956+0000 mgr.y (mgr.24272) 158 : cluster [DBG] pgmap v133: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:51:45.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:51:44 smithi134 bash[24068]: audit 2024-07-15T06:51:44.193578+0000 mon.a (mon.0) 890 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:51:45.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:51:44 smithi089 bash[31372]: audit 2024-07-15T06:51:44.193578+0000 mon.a (mon.0) 890 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:51:45.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:51:44 smithi089 bash[21793]: audit 2024-07-15T06:51:44.193578+0000 mon.a (mon.0) 890 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:51:45.773 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:51:45 smithi089 bash[31372]: cluster 2024-07-15T06:51:44.179495+0000 mgr.y (mgr.24272) 159 : cluster [DBG] pgmap v134: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:51:45.773 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:51:45 smithi089 bash[21793]: cluster 2024-07-15T06:51:44.179495+0000 mgr.y (mgr.24272) 159 : cluster [DBG] pgmap v134: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:51:46.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:51:45 smithi134 bash[24068]: cluster 2024-07-15T06:51:44.179495+0000 mgr.y (mgr.24272) 159 : cluster [DBG] pgmap v134: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:51:47.304 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:51:46 smithi134 bash[25019]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:51:46] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-07-15T06:51:48.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:51:47 smithi134 bash[24068]: cluster 2024-07-15T06:51:46.180579+0000 mgr.y (mgr.24272) 160 : cluster [DBG] pgmap v135: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:51:48.071 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:51:47 smithi089 bash[21793]: cluster 2024-07-15T06:51:46.180579+0000 mgr.y (mgr.24272) 160 : cluster [DBG] pgmap v135: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:51:48.071 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:51:47 smithi089 bash[31372]: cluster 2024-07-15T06:51:46.180579+0000 mgr.y (mgr.24272) 160 : cluster [DBG] pgmap v135: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:51:50.014 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:51:49 smithi089 bash[21793]: cluster 2024-07-15T06:51:48.180996+0000 mgr.y (mgr.24272) 161 : cluster [DBG] pgmap v136: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:51:50.015 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:51:49 smithi089 bash[31372]: cluster 2024-07-15T06:51:48.180996+0000 mgr.y (mgr.24272) 161 : cluster [DBG] pgmap v136: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:51:50.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:51:49 smithi134 bash[24068]: cluster 2024-07-15T06:51:48.180996+0000 mgr.y (mgr.24272) 161 : cluster [DBG] pgmap v136: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:51:52.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:51:51 smithi134 bash[24068]: cluster 2024-07-15T06:51:50.181513+0000 mgr.y (mgr.24272) 162 : cluster [DBG] pgmap v137: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:51:52.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:51:51 smithi089 bash[21793]: cluster 2024-07-15T06:51:50.181513+0000 mgr.y (mgr.24272) 162 : cluster [DBG] pgmap v137: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:51:52.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:51:51 smithi089 bash[31372]: cluster 2024-07-15T06:51:50.181513+0000 mgr.y (mgr.24272) 162 : cluster [DBG] pgmap v137: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:51:52.947 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:51:52 smithi089 bash[22045]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:51:52] "GET /metrics HTTP/1.1" 200 239257 "" "Prometheus/2.43.0" 2024-07-15T06:51:54.045 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:51:53 smithi089 bash[31372]: cluster 2024-07-15T06:51:52.182094+0000 mgr.y (mgr.24272) 163 : cluster [DBG] pgmap v138: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:51:54.045 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:51:53 smithi089 bash[21793]: cluster 2024-07-15T06:51:52.182094+0000 mgr.y (mgr.24272) 163 : cluster [DBG] pgmap v138: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:51:54.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:51:53 smithi134 bash[24068]: cluster 2024-07-15T06:51:52.182094+0000 mgr.y (mgr.24272) 163 : cluster [DBG] pgmap v138: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:51:55.570 INFO:teuthology.orchestra.run.smithi089.stdout:107374182507 2024-07-15T06:51:55.869 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:51:55 smithi089 bash[21793]: cluster 2024-07-15T06:51:54.182672+0000 mgr.y (mgr.24272) 164 : cluster [DBG] pgmap v139: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:51:55.869 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:51:55 smithi089 bash[21793]: audit 2024-07-15T06:51:55.576479+0000 mon.c (mon.2) 26 : audit [DBG] from='client.? 172.21.15.89:0/3701334467' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-07-15T06:51:55.869 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:51:55 smithi089 bash[31372]: cluster 2024-07-15T06:51:54.182672+0000 mgr.y (mgr.24272) 164 : cluster [DBG] pgmap v139: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:51:55.869 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:51:55 smithi089 bash[31372]: audit 2024-07-15T06:51:55.576479+0000 mon.c (mon.2) 26 : audit [DBG] from='client.? 172.21.15.89:0/3701334467' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-07-15T06:51:56.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:51:55 smithi134 bash[24068]: cluster 2024-07-15T06:51:54.182672+0000 mgr.y (mgr.24272) 164 : cluster [DBG] pgmap v139: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:51:56.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:51:55 smithi134 bash[24068]: audit 2024-07-15T06:51:55.576479+0000 mon.c (mon.2) 26 : audit [DBG] from='client.? 172.21.15.89:0/3701334467' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-07-15T06:51:56.577 INFO:teuthology.orchestra.run.smithi089.stdout:231928234049 2024-07-15T06:51:56.800 INFO:teuthology.orchestra.run.smithi089.stdout:34359738507 2024-07-15T06:51:56.803 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:51:56 smithi089 bash[21793]: audit 2024-07-15T06:51:56.582976+0000 mon.a (mon.0) 891 : audit [DBG] from='client.? 172.21.15.89:0/1509570632' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-07-15T06:51:56.803 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:51:56 smithi089 bash[31372]: audit 2024-07-15T06:51:56.582976+0000 mon.a (mon.0) 891 : audit [DBG] from='client.? 172.21.15.89:0/1509570632' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-07-15T06:51:57.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:51:56 smithi134 bash[24068]: audit 2024-07-15T06:51:56.582976+0000 mon.a (mon.0) 891 : audit [DBG] from='client.? 172.21.15.89:0/1509570632' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-07-15T06:51:57.054 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:51:56 smithi134 bash[25019]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:51:56] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-07-15T06:51:57.655 INFO:tasks.cephadm.ceph_manager.ceph:need seq 107374182503 got 107374182507 for osd.3 2024-07-15T06:51:57.655 DEBUG:teuthology.parallel:result is None 2024-07-15T06:51:57.911 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:51:57 smithi089 bash[21793]: cluster 2024-07-15T06:51:56.183770+0000 mgr.y (mgr.24272) 165 : cluster [DBG] pgmap v140: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:51:57.911 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:51:57 smithi089 bash[21793]: audit 2024-07-15T06:51:56.805948+0000 mon.a (mon.0) 892 : audit [DBG] from='client.? 172.21.15.89:0/1162745016' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-07-15T06:51:57.911 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:51:57 smithi089 bash[31372]: cluster 2024-07-15T06:51:56.183770+0000 mgr.y (mgr.24272) 165 : cluster [DBG] pgmap v140: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:51:57.911 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:51:57 smithi089 bash[31372]: audit 2024-07-15T06:51:56.805948+0000 mon.a (mon.0) 892 : audit [DBG] from='client.? 172.21.15.89:0/1162745016' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-07-15T06:51:58.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:51:57 smithi134 bash[24068]: cluster 2024-07-15T06:51:56.183770+0000 mgr.y (mgr.24272) 165 : cluster [DBG] pgmap v140: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:51:58.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:51:57 smithi134 bash[24068]: audit 2024-07-15T06:51:56.805948+0000 mon.a (mon.0) 892 : audit [DBG] from='client.? 172.21.15.89:0/1162745016' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-07-15T06:51:58.207 INFO:teuthology.orchestra.run.smithi089.stdout:163208757334 2024-07-15T06:51:58.529 INFO:teuthology.orchestra.run.smithi089.stdout:77309411445 2024-07-15T06:51:58.766 INFO:teuthology.orchestra.run.smithi089.stdout:128849018977 2024-07-15T06:51:59.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:51:58 smithi134 bash[24068]: audit 2024-07-15T06:51:58.213321+0000 mon.c (mon.2) 27 : audit [DBG] from='client.? 172.21.15.89:0/2971220166' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-07-15T06:51:59.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:51:58 smithi134 bash[24068]: audit 2024-07-15T06:51:58.535454+0000 mon.a (mon.0) 893 : audit [DBG] from='client.? 172.21.15.89:0/3284174695' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-07-15T06:51:59.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:51:58 smithi089 bash[21793]: audit 2024-07-15T06:51:58.213321+0000 mon.c (mon.2) 27 : audit [DBG] from='client.? 172.21.15.89:0/2971220166' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-07-15T06:51:59.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:51:58 smithi089 bash[21793]: audit 2024-07-15T06:51:58.535454+0000 mon.a (mon.0) 893 : audit [DBG] from='client.? 172.21.15.89:0/3284174695' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-07-15T06:51:59.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:51:58 smithi089 bash[31372]: audit 2024-07-15T06:51:58.213321+0000 mon.c (mon.2) 27 : audit [DBG] from='client.? 172.21.15.89:0/2971220166' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-07-15T06:51:59.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:51:58 smithi089 bash[31372]: audit 2024-07-15T06:51:58.535454+0000 mon.a (mon.0) 893 : audit [DBG] from='client.? 172.21.15.89:0/3284174695' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-07-15T06:51:59.868 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:51:59 smithi089 bash[31372]: cluster 2024-07-15T06:51:58.184270+0000 mgr.y (mgr.24272) 166 : cluster [DBG] pgmap v141: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:51:59.869 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:51:59 smithi089 bash[31372]: audit 2024-07-15T06:51:58.772454+0000 mon.a (mon.0) 894 : audit [DBG] from='client.? 172.21.15.89:0/2964565226' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-07-15T06:51:59.869 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:51:59 smithi089 bash[31372]: audit 2024-07-15T06:51:59.193893+0000 mon.a (mon.0) 895 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:51:59.869 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:51:59 smithi089 bash[21793]: cluster 2024-07-15T06:51:58.184270+0000 mgr.y (mgr.24272) 166 : cluster [DBG] pgmap v141: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:51:59.869 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:51:59 smithi089 bash[21793]: audit 2024-07-15T06:51:58.772454+0000 mon.a (mon.0) 894 : audit [DBG] from='client.? 172.21.15.89:0/2964565226' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-07-15T06:51:59.869 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:51:59 smithi089 bash[21793]: audit 2024-07-15T06:51:59.193893+0000 mon.a (mon.0) 895 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:51:59.967 INFO:teuthology.orchestra.run.smithi089.stdout:55834574976 2024-07-15T06:52:00.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:51:59 smithi134 bash[24068]: cluster 2024-07-15T06:51:58.184270+0000 mgr.y (mgr.24272) 166 : cluster [DBG] pgmap v141: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:52:00.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:51:59 smithi134 bash[24068]: audit 2024-07-15T06:51:58.772454+0000 mon.a (mon.0) 894 : audit [DBG] from='client.? 172.21.15.89:0/2964565226' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-07-15T06:52:00.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:51:59 smithi134 bash[24068]: audit 2024-07-15T06:51:59.193893+0000 mon.a (mon.0) 895 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:52:00.150 INFO:teuthology.orchestra.run.smithi089.stdout:197568495692 2024-07-15T06:52:00.284 INFO:tasks.cephadm.ceph_manager.ceph:need seq 163208757329 got 163208757334 for osd.5 2024-07-15T06:52:00.284 DEBUG:teuthology.parallel:result is None 2024-07-15T06:52:00.338 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738502 got 34359738507 for osd.0 2024-07-15T06:52:00.338 DEBUG:teuthology.parallel:result is None 2024-07-15T06:52:00.341 INFO:tasks.cephadm.ceph_manager.ceph:need seq 231928234045 got 231928234049 for osd.7 2024-07-15T06:52:00.341 DEBUG:teuthology.parallel:result is None 2024-07-15T06:52:01.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:52:00 smithi134 bash[24068]: audit 2024-07-15T06:51:59.972992+0000 mon.c (mon.2) 28 : audit [DBG] from='client.? 172.21.15.89:0/1465925057' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-07-15T06:52:01.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:52:00 smithi134 bash[24068]: audit 2024-07-15T06:52:00.156705+0000 mon.c (mon.2) 29 : audit [DBG] from='client.? 172.21.15.89:0/2176774456' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-07-15T06:52:01.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:52:00 smithi089 bash[21793]: audit 2024-07-15T06:51:59.972992+0000 mon.c (mon.2) 28 : audit [DBG] from='client.? 172.21.15.89:0/1465925057' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-07-15T06:52:01.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:52:00 smithi089 bash[21793]: audit 2024-07-15T06:52:00.156705+0000 mon.c (mon.2) 29 : audit [DBG] from='client.? 172.21.15.89:0/2176774456' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-07-15T06:52:01.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:52:00 smithi089 bash[31372]: audit 2024-07-15T06:51:59.972992+0000 mon.c (mon.2) 28 : audit [DBG] from='client.? 172.21.15.89:0/1465925057' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-07-15T06:52:01.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:52:00 smithi089 bash[31372]: audit 2024-07-15T06:52:00.156705+0000 mon.c (mon.2) 29 : audit [DBG] from='client.? 172.21.15.89:0/2176774456' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-07-15T06:52:01.394 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411440 got 77309411445 for osd.2 2024-07-15T06:52:01.394 DEBUG:teuthology.parallel:result is None 2024-07-15T06:52:01.827 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849018973 got 128849018977 for osd.4 2024-07-15T06:52:01.827 DEBUG:teuthology.parallel:result is None 2024-07-15T06:52:02.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:52:01 smithi134 bash[24068]: cluster 2024-07-15T06:52:00.185059+0000 mgr.y (mgr.24272) 167 : cluster [DBG] pgmap v142: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:52:02.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:52:01 smithi089 bash[31372]: cluster 2024-07-15T06:52:00.185059+0000 mgr.y (mgr.24272) 167 : cluster [DBG] pgmap v142: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:52:02.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:52:01 smithi089 bash[21793]: cluster 2024-07-15T06:52:00.185059+0000 mgr.y (mgr.24272) 167 : cluster [DBG] pgmap v142: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:52:02.186 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574972 got 55834574976 for osd.1 2024-07-15T06:52:02.186 DEBUG:teuthology.parallel:result is None 2024-07-15T06:52:02.228 INFO:tasks.cephadm.ceph_manager.ceph:need seq 197568495688 got 197568495692 for osd.6 2024-07-15T06:52:02.228 DEBUG:teuthology.parallel:result is None 2024-07-15T06:52:02.228 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-07-15T06:52:02.228 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph pg dump --format=json 2024-07-15T06:52:03.119 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:52:02 smithi089 bash[22045]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:52:02] "GET /metrics HTTP/1.1" 200 239255 "" "Prometheus/2.43.0" 2024-07-15T06:52:04.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:52:03 smithi134 bash[24068]: cluster 2024-07-15T06:52:02.185623+0000 mgr.y (mgr.24272) 168 : cluster [DBG] pgmap v143: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:52:04.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:52:03 smithi089 bash[21793]: cluster 2024-07-15T06:52:02.185623+0000 mgr.y (mgr.24272) 168 : cluster [DBG] pgmap v143: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:52:04.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:52:03 smithi089 bash[31372]: cluster 2024-07-15T06:52:02.185623+0000 mgr.y (mgr.24272) 168 : cluster [DBG] pgmap v143: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:52:06.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:52:05 smithi134 bash[24068]: cluster 2024-07-15T06:52:04.186242+0000 mgr.y (mgr.24272) 169 : cluster [DBG] pgmap v144: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:52:06.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:52:05 smithi089 bash[21793]: cluster 2024-07-15T06:52:04.186242+0000 mgr.y (mgr.24272) 169 : cluster [DBG] pgmap v144: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:52:06.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:52:05 smithi089 bash[31372]: cluster 2024-07-15T06:52:04.186242+0000 mgr.y (mgr.24272) 169 : cluster [DBG] pgmap v144: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:52:07.019 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T06:52:07.304 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:52:06 smithi134 bash[25019]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:52:06] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-07-15T06:52:08.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:52:07 smithi134 bash[24068]: cluster 2024-07-15T06:52:06.187331+0000 mgr.y (mgr.24272) 170 : cluster [DBG] pgmap v145: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:52:08.118 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:52:07 smithi089 bash[21793]: cluster 2024-07-15T06:52:06.187331+0000 mgr.y (mgr.24272) 170 : cluster [DBG] pgmap v145: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:52:08.118 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:52:07 smithi089 bash[31372]: cluster 2024-07-15T06:52:06.187331+0000 mgr.y (mgr.24272) 170 : cluster [DBG] pgmap v145: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:52:10.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:52:09 smithi134 bash[24068]: cluster 2024-07-15T06:52:08.187722+0000 mgr.y (mgr.24272) 171 : cluster [DBG] pgmap v146: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:52:10.060 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:52:09 smithi089 bash[31372]: cluster 2024-07-15T06:52:08.187722+0000 mgr.y (mgr.24272) 171 : cluster [DBG] pgmap v146: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:52:10.061 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:52:09 smithi089 bash[21793]: cluster 2024-07-15T06:52:08.187722+0000 mgr.y (mgr.24272) 171 : cluster [DBG] pgmap v146: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:52:10.399 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-07-15T06:52:10.400 INFO:teuthology.orchestra.run.smithi089.stderr:dumped all 2024-07-15T06:52:11.120 INFO:teuthology.orchestra.run.smithi089.stdout:{"pg_ready":true,"pg_map":{"version":147,"stamp":"2024-07-15T06:52:10.187907+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592706,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":259,"num_read_kb":355,"num_write":224,"num_write_kb":2429,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":173,"ondisk_log_size":173,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2386292,"kb_used_data":7428,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747574924,"statfs":{"total":767960285184,"available":765516722176,"internally_reserved":0,"allocated":7606272,"data_stored":4286160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003889"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.803509+0000","last_change":"2024-07-15T06:50:29.852535+0000","last_active":"2024-07-15T06:50:44.803509+0000","last_peered":"2024-07-15T06:50:44.803509+0000","last_clean":"2024-07-15T06:50:44.803509+0000","last_became_active":"2024-07-15T06:50:29.851305+0000","last_became_peered":"2024-07-15T06:50:29.851305+0000","last_unstale":"2024-07-15T06:50:44.803509+0000","last_undegraded":"2024-07-15T06:50:44.803509+0000","last_fullsized":"2024-07-15T06:50:44.803509+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T11:34:38.540931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":18,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.804032+0000","last_change":"2024-07-15T06:50:29.262370+0000","last_active":"2024-07-15T06:50:44.804032+0000","last_peered":"2024-07-15T06:50:44.804032+0000","last_clean":"2024-07-15T06:50:44.804032+0000","last_became_active":"2024-07-15T06:50:29.261755+0000","last_became_peered":"2024-07-15T06:50:29.261755+0000","last_unstale":"2024-07-15T06:50:44.804032+0000","last_undegraded":"2024-07-15T06:50:44.804032+0000","last_fullsized":"2024-07-15T06:50:44.804032+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T17:22:35.393499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":7,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.790474+0000","last_change":"2024-07-15T06:50:29.851103+0000","last_active":"2024-07-15T06:50:44.790474+0000","last_peered":"2024-07-15T06:50:44.790474+0000","last_clean":"2024-07-15T06:50:44.790474+0000","last_became_active":"2024-07-15T06:50:29.850222+0000","last_became_peered":"2024-07-15T06:50:29.850222+0000","last_unstale":"2024-07-15T06:50:44.790474+0000","last_undegraded":"2024-07-15T06:50:44.790474+0000","last_fullsized":"2024-07-15T06:50:44.790474+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T09:44:30.500327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.790700+0000","last_change":"2024-07-15T06:50:29.275925+0000","last_active":"2024-07-15T06:50:44.790700+0000","last_peered":"2024-07-15T06:50:44.790700+0000","last_clean":"2024-07-15T06:50:44.790700+0000","last_became_active":"2024-07-15T06:50:29.275169+0000","last_became_peered":"2024-07-15T06:50:29.275169+0000","last_unstale":"2024-07-15T06:50:44.790700+0000","last_undegraded":"2024-07-15T06:50:44.790700+0000","last_fullsized":"2024-07-15T06:50:44.790700+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T17:59:53.279149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.790532+0000","last_change":"2024-07-15T06:50:29.852471+0000","last_active":"2024-07-15T06:50:44.790532+0000","last_peered":"2024-07-15T06:50:44.790532+0000","last_clean":"2024-07-15T06:50:44.790532+0000","last_became_active":"2024-07-15T06:50:29.851082+0000","last_became_peered":"2024-07-15T06:50:29.851082+0000","last_unstale":"2024-07-15T06:50:44.790532+0000","last_undegraded":"2024-07-15T06:50:44.790532+0000","last_fullsized":"2024-07-15T06:50:44.790532+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T14:46:50.862472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.803607+0000","last_change":"2024-07-15T06:50:29.262044+0000","last_active":"2024-07-15T06:50:44.803607+0000","last_peered":"2024-07-15T06:50:44.803607+0000","last_clean":"2024-07-15T06:50:44.803607+0000","last_became_active":"2024-07-15T06:50:29.261056+0000","last_became_peered":"2024-07-15T06:50:29.261056+0000","last_unstale":"2024-07-15T06:50:44.803607+0000","last_undegraded":"2024-07-15T06:50:44.803607+0000","last_fullsized":"2024-07-15T06:50:44.803607+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T08:13:43.705862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.803686+0000","last_change":"2024-07-15T06:50:29.849048+0000","last_active":"2024-07-15T06:50:44.803686+0000","last_peered":"2024-07-15T06:50:44.803686+0000","last_clean":"2024-07-15T06:50:44.803686+0000","last_became_active":"2024-07-15T06:50:29.848064+0000","last_became_peered":"2024-07-15T06:50:29.848064+0000","last_unstale":"2024-07-15T06:50:44.803686+0000","last_undegraded":"2024-07-15T06:50:44.803686+0000","last_fullsized":"2024-07-15T06:50:44.803686+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T17:10:05.793024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":26,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.797343+0000","last_change":"2024-07-15T06:50:29.251566+0000","last_active":"2024-07-15T06:50:44.797343+0000","last_peered":"2024-07-15T06:50:44.797343+0000","last_clean":"2024-07-15T06:50:44.797343+0000","last_became_active":"2024-07-15T06:50:29.250951+0000","last_became_peered":"2024-07-15T06:50:29.250951+0000","last_unstale":"2024-07-15T06:50:44.797343+0000","last_undegraded":"2024-07-15T06:50:44.797343+0000","last_fullsized":"2024-07-15T06:50:44.797343+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T15:44:37.697435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":12,"num_read_kb":12,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":16,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.804756+0000","last_change":"2024-07-15T06:50:29.269211+0000","last_active":"2024-07-15T06:50:44.804756+0000","last_peered":"2024-07-15T06:50:44.804756+0000","last_clean":"2024-07-15T06:50:44.804756+0000","last_became_active":"2024-07-15T06:50:29.268466+0000","last_became_peered":"2024-07-15T06:50:29.268466+0000","last_unstale":"2024-07-15T06:50:44.804756+0000","last_undegraded":"2024-07-15T06:50:44.804756+0000","last_fullsized":"2024-07-15T06:50:44.804756+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T10:28:35.703616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.790094+0000","last_change":"2024-07-15T06:50:29.845162+0000","last_active":"2024-07-15T06:50:44.790094+0000","last_peered":"2024-07-15T06:50:44.790094+0000","last_clean":"2024-07-15T06:50:44.790094+0000","last_became_active":"2024-07-15T06:50:29.844204+0000","last_became_peered":"2024-07-15T06:50:29.844204+0000","last_unstale":"2024-07-15T06:50:44.790094+0000","last_undegraded":"2024-07-15T06:50:44.790094+0000","last_fullsized":"2024-07-15T06:50:44.790094+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T16:56:12.605992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":17,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.804896+0000","last_change":"2024-07-15T06:50:29.273685+0000","last_active":"2024-07-15T06:50:44.804896+0000","last_peered":"2024-07-15T06:50:44.804896+0000","last_clean":"2024-07-15T06:50:44.804896+0000","last_became_active":"2024-07-15T06:50:29.272972+0000","last_became_peered":"2024-07-15T06:50:29.272972+0000","last_unstale":"2024-07-15T06:50:44.804896+0000","last_undegraded":"2024-07-15T06:50:44.804896+0000","last_fullsized":"2024-07-15T06:50:44.804896+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T08:42:09.115844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.797509+0000","last_change":"2024-07-15T06:50:29.303167+0000","last_active":"2024-07-15T06:50:44.797509+0000","last_peered":"2024-07-15T06:50:44.797509+0000","last_clean":"2024-07-15T06:50:44.797509+0000","last_became_active":"2024-07-15T06:50:29.302500+0000","last_became_peered":"2024-07-15T06:50:29.302500+0000","last_unstale":"2024-07-15T06:50:44.797509+0000","last_undegraded":"2024-07-15T06:50:44.797509+0000","last_fullsized":"2024-07-15T06:50:44.797509+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T12:32:35.025536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":16,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.804760+0000","last_change":"2024-07-15T06:50:29.271443+0000","last_active":"2024-07-15T06:50:44.804760+0000","last_peered":"2024-07-15T06:50:44.804760+0000","last_clean":"2024-07-15T06:50:44.804760+0000","last_became_active":"2024-07-15T06:50:29.270851+0000","last_became_peered":"2024-07-15T06:50:29.270851+0000","last_unstale":"2024-07-15T06:50:44.804760+0000","last_undegraded":"2024-07-15T06:50:44.804760+0000","last_fullsized":"2024-07-15T06:50:44.804760+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T14:45:12.733929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":23,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-07-15T06:50:45.797074+0000","last_change":"2024-07-15T06:50:45.797074+0000","last_active":"2024-07-15T06:50:45.797074+0000","last_peered":"2024-07-15T06:50:45.797074+0000","last_clean":"2024-07-15T06:50:45.797074+0000","last_became_active":"2024-07-15T06:50:45.796066+0000","last_became_peered":"2024-07-15T06:50:45.796066+0000","last_unstale":"2024-07-15T06:50:45.797074+0000","last_undegraded":"2024-07-15T06:50:45.797074+0000","last_fullsized":"2024-07-15T06:50:45.797074+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T09:53:48.422246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.790646+0000","last_change":"2024-07-15T06:50:29.274245+0000","last_active":"2024-07-15T06:50:44.790646+0000","last_peered":"2024-07-15T06:50:44.790646+0000","last_clean":"2024-07-15T06:50:44.790646+0000","last_became_active":"2024-07-15T06:50:29.273642+0000","last_became_peered":"2024-07-15T06:50:29.273642+0000","last_unstale":"2024-07-15T06:50:44.790646+0000","last_undegraded":"2024-07-15T06:50:44.790646+0000","last_fullsized":"2024-07-15T06:50:44.790646+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T09:06:19.595465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.790771+0000","last_change":"2024-07-15T06:50:29.304844+0000","last_active":"2024-07-15T06:50:44.790771+0000","last_peered":"2024-07-15T06:50:44.790771+0000","last_clean":"2024-07-15T06:50:44.790771+0000","last_became_active":"2024-07-15T06:50:29.304368+0000","last_became_peered":"2024-07-15T06:50:29.304368+0000","last_unstale":"2024-07-15T06:50:44.790771+0000","last_undegraded":"2024-07-15T06:50:44.790771+0000","last_fullsized":"2024-07-15T06:50:44.790771+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T07:52:24.418782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"68'3","reported_seq":18,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.802899+0000","last_change":"2024-07-15T06:50:29.269551+0000","last_active":"2024-07-15T06:50:44.802899+0000","last_peered":"2024-07-15T06:50:44.802899+0000","last_clean":"2024-07-15T06:50:44.802899+0000","last_became_active":"2024-07-15T06:50:29.268783+0000","last_became_peered":"2024-07-15T06:50:29.268783+0000","last_unstale":"2024-07-15T06:50:44.802899+0000","last_undegraded":"2024-07-15T06:50:44.802899+0000","last_fullsized":"2024-07-15T06:50:44.802899+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T06:57:29.304742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.798055+0000","last_change":"2024-07-15T06:50:29.299475+0000","last_active":"2024-07-15T06:50:44.798055+0000","last_peered":"2024-07-15T06:50:44.798055+0000","last_clean":"2024-07-15T06:50:44.798055+0000","last_became_active":"2024-07-15T06:50:29.298804+0000","last_became_peered":"2024-07-15T06:50:29.298804+0000","last_unstale":"2024-07-15T06:50:44.798055+0000","last_undegraded":"2024-07-15T06:50:44.798055+0000","last_fullsized":"2024-07-15T06:50:44.798055+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T11:01:29.878472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.800793+0000","last_change":"2024-07-15T06:50:29.254893+0000","last_active":"2024-07-15T06:50:44.800793+0000","last_peered":"2024-07-15T06:50:44.800793+0000","last_clean":"2024-07-15T06:50:44.800793+0000","last_became_active":"2024-07-15T06:50:29.254025+0000","last_became_peered":"2024-07-15T06:50:29.254025+0000","last_unstale":"2024-07-15T06:50:44.800793+0000","last_undegraded":"2024-07-15T06:50:44.800793+0000","last_fullsized":"2024-07-15T06:50:44.800793+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T16:38:14.132555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.798543+0000","last_change":"2024-07-15T06:50:29.301409+0000","last_active":"2024-07-15T06:50:44.798543+0000","last_peered":"2024-07-15T06:50:44.798543+0000","last_clean":"2024-07-15T06:50:44.798543+0000","last_became_active":"2024-07-15T06:50:29.300988+0000","last_became_peered":"2024-07-15T06:50:29.300988+0000","last_unstale":"2024-07-15T06:50:44.798543+0000","last_undegraded":"2024-07-15T06:50:44.798543+0000","last_fullsized":"2024-07-15T06:50:44.798543+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T14:40:46.878950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.797901+0000","last_change":"2024-07-15T06:50:29.257021+0000","last_active":"2024-07-15T06:50:44.797901+0000","last_peered":"2024-07-15T06:50:44.797901+0000","last_clean":"2024-07-15T06:50:44.797901+0000","last_became_active":"2024-07-15T06:50:29.256202+0000","last_became_peered":"2024-07-15T06:50:29.256202+0000","last_unstale":"2024-07-15T06:50:44.797901+0000","last_undegraded":"2024-07-15T06:50:44.797901+0000","last_fullsized":"2024-07-15T06:50:44.797901+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T18:27:17.492902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.798748+0000","last_change":"2024-07-15T06:50:29.302450+0000","last_active":"2024-07-15T06:50:44.798748+0000","last_peered":"2024-07-15T06:50:44.798748+0000","last_clean":"2024-07-15T06:50:44.798748+0000","last_became_active":"2024-07-15T06:50:29.302017+0000","last_became_peered":"2024-07-15T06:50:29.302017+0000","last_unstale":"2024-07-15T06:50:44.798748+0000","last_undegraded":"2024-07-15T06:50:44.798748+0000","last_fullsized":"2024-07-15T06:50:44.798748+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T15:31:08.157676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.802186+0000","last_change":"2024-07-15T06:50:29.277528+0000","last_active":"2024-07-15T06:50:44.802186+0000","last_peered":"2024-07-15T06:50:44.802186+0000","last_clean":"2024-07-15T06:50:44.802186+0000","last_became_active":"2024-07-15T06:50:29.277057+0000","last_became_peered":"2024-07-15T06:50:29.277057+0000","last_unstale":"2024-07-15T06:50:44.802186+0000","last_undegraded":"2024-07-15T06:50:44.802186+0000","last_fullsized":"2024-07-15T06:50:44.802186+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T09:04:06.474542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.795540+0000","last_change":"2024-07-15T06:50:29.311569+0000","last_active":"2024-07-15T06:50:44.795540+0000","last_peered":"2024-07-15T06:50:44.795540+0000","last_clean":"2024-07-15T06:50:44.795540+0000","last_became_active":"2024-07-15T06:50:29.311002+0000","last_became_peered":"2024-07-15T06:50:29.311002+0000","last_unstale":"2024-07-15T06:50:44.795540+0000","last_undegraded":"2024-07-15T06:50:44.795540+0000","last_fullsized":"2024-07-15T06:50:44.795540+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T17:51:08.575282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"59'128","reported_seq":277,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.803597+0000","last_change":"2024-07-15T06:46:49.356522+0000","last_active":"2024-07-15T06:50:44.803597+0000","last_peered":"2024-07-15T06:50:44.803597+0000","last_clean":"2024-07-15T06:50:44.803597+0000","last_became_active":"2024-07-15T06:46:49.355501+0000","last_became_peered":"2024-07-15T06:46:49.355501+0000","last_unstale":"2024-07-15T06:50:44.803597+0000","last_undegraded":"2024-07-15T06:50:44.803597+0000","last_fullsized":"2024-07-15T06:50:44.803597+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:42:21.582843+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:42:21.582843+0000","last_clean_scrub_stamp":"2024-07-15T06:42:21.582843+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T10:17:36.460676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":18,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.790563+0000","last_change":"2024-07-15T06:48:28.672045+0000","last_active":"2024-07-15T06:50:44.790563+0000","last_peered":"2024-07-15T06:50:44.790563+0000","last_clean":"2024-07-15T06:50:44.790563+0000","last_became_active":"2024-07-15T06:48:28.671290+0000","last_became_peered":"2024-07-15T06:48:28.671290+0000","last_unstale":"2024-07-15T06:50:44.790563+0000","last_undegraded":"2024-07-15T06:50:44.790563+0000","last_fullsized":"2024-07-15T06:50:44.790563+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_clean_scrub_stamp":"2024-07-15T06:48:27.655327+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T11:10:59.552676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.802607+0000","last_change":"2024-07-15T06:50:29.277850+0000","last_active":"2024-07-15T06:50:44.802607+0000","last_peered":"2024-07-15T06:50:44.802607+0000","last_clean":"2024-07-15T06:50:44.802607+0000","last_became_active":"2024-07-15T06:50:29.276384+0000","last_became_peered":"2024-07-15T06:50:29.276384+0000","last_unstale":"2024-07-15T06:50:44.802607+0000","last_undegraded":"2024-07-15T06:50:44.802607+0000","last_fullsized":"2024-07-15T06:50:44.802607+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T15:57:11.793888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.801412+0000","last_change":"2024-07-15T06:50:29.851397+0000","last_active":"2024-07-15T06:50:44.801412+0000","last_peered":"2024-07-15T06:50:44.801412+0000","last_clean":"2024-07-15T06:50:44.801412+0000","last_became_active":"2024-07-15T06:50:29.850009+0000","last_became_peered":"2024-07-15T06:50:29.850009+0000","last_unstale":"2024-07-15T06:50:44.801412+0000","last_undegraded":"2024-07-15T06:50:44.801412+0000","last_fullsized":"2024-07-15T06:50:44.801412+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T08:04:19.006397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":18,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.803453+0000","last_change":"2024-07-15T06:48:28.737091+0000","last_active":"2024-07-15T06:50:44.803453+0000","last_peered":"2024-07-15T06:50:44.803453+0000","last_clean":"2024-07-15T06:50:44.803453+0000","last_became_active":"2024-07-15T06:48:28.736574+0000","last_became_peered":"2024-07-15T06:48:28.736574+0000","last_unstale":"2024-07-15T06:50:44.803453+0000","last_undegraded":"2024-07-15T06:50:44.803453+0000","last_fullsized":"2024-07-15T06:50:44.803453+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_clean_scrub_stamp":"2024-07-15T06:48:27.655327+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T09:31:07.195677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.789928+0000","last_change":"2024-07-15T06:50:29.261545+0000","last_active":"2024-07-15T06:50:44.789928+0000","last_peered":"2024-07-15T06:50:44.789928+0000","last_clean":"2024-07-15T06:50:44.789928+0000","last_became_active":"2024-07-15T06:50:29.261213+0000","last_became_peered":"2024-07-15T06:50:29.261213+0000","last_unstale":"2024-07-15T06:50:44.789928+0000","last_undegraded":"2024-07-15T06:50:44.789928+0000","last_fullsized":"2024-07-15T06:50:44.789928+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T10:34:24.671609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.790362+0000","last_change":"2024-07-15T06:50:29.303396+0000","last_active":"2024-07-15T06:50:44.790362+0000","last_peered":"2024-07-15T06:50:44.790362+0000","last_clean":"2024-07-15T06:50:44.790362+0000","last_became_active":"2024-07-15T06:50:29.302950+0000","last_became_peered":"2024-07-15T06:50:29.302950+0000","last_unstale":"2024-07-15T06:50:44.790362+0000","last_undegraded":"2024-07-15T06:50:44.790362+0000","last_fullsized":"2024-07-15T06:50:44.790362+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T09:41:19.686254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":18,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.798119+0000","last_change":"2024-07-15T06:48:28.738441+0000","last_active":"2024-07-15T06:50:44.798119+0000","last_peered":"2024-07-15T06:50:44.798119+0000","last_clean":"2024-07-15T06:50:44.798119+0000","last_became_active":"2024-07-15T06:48:28.737053+0000","last_became_peered":"2024-07-15T06:48:28.737053+0000","last_unstale":"2024-07-15T06:50:44.798119+0000","last_undegraded":"2024-07-15T06:50:44.798119+0000","last_fullsized":"2024-07-15T06:50:44.798119+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_clean_scrub_stamp":"2024-07-15T06:48:27.655327+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T08:33:33.092342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.792499+0000","last_change":"2024-07-15T06:50:29.249905+0000","last_active":"2024-07-15T06:50:44.792499+0000","last_peered":"2024-07-15T06:50:44.792499+0000","last_clean":"2024-07-15T06:50:44.792499+0000","last_became_active":"2024-07-15T06:50:29.249163+0000","last_became_peered":"2024-07-15T06:50:29.249163+0000","last_unstale":"2024-07-15T06:50:44.792499+0000","last_undegraded":"2024-07-15T06:50:44.792499+0000","last_fullsized":"2024-07-15T06:50:44.792499+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T11:48:19.218891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.795525+0000","last_change":"2024-07-15T06:50:29.304233+0000","last_active":"2024-07-15T06:50:44.795525+0000","last_peered":"2024-07-15T06:50:44.795525+0000","last_clean":"2024-07-15T06:50:44.795525+0000","last_became_active":"2024-07-15T06:50:29.303966+0000","last_became_peered":"2024-07-15T06:50:29.303966+0000","last_unstale":"2024-07-15T06:50:44.795525+0000","last_undegraded":"2024-07-15T06:50:44.795525+0000","last_fullsized":"2024-07-15T06:50:44.795525+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T17:22:58.514985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":18,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.790222+0000","last_change":"2024-07-15T06:48:28.674943+0000","last_active":"2024-07-15T06:50:44.790222+0000","last_peered":"2024-07-15T06:50:44.790222+0000","last_clean":"2024-07-15T06:50:44.790222+0000","last_became_active":"2024-07-15T06:48:28.674174+0000","last_became_peered":"2024-07-15T06:48:28.674174+0000","last_unstale":"2024-07-15T06:50:44.790222+0000","last_undegraded":"2024-07-15T06:50:44.790222+0000","last_fullsized":"2024-07-15T06:50:44.790222+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_clean_scrub_stamp":"2024-07-15T06:48:27.655327+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T17:40:56.394801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.800666+0000","last_change":"2024-07-15T06:50:29.258941+0000","last_active":"2024-07-15T06:50:44.800666+0000","last_peered":"2024-07-15T06:50:44.800666+0000","last_clean":"2024-07-15T06:50:44.800666+0000","last_became_active":"2024-07-15T06:50:29.258207+0000","last_became_peered":"2024-07-15T06:50:29.258207+0000","last_unstale":"2024-07-15T06:50:44.800666+0000","last_undegraded":"2024-07-15T06:50:44.800666+0000","last_fullsized":"2024-07-15T06:50:44.800666+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T14:30:34.638511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.794413+0000","last_change":"2024-07-15T06:50:29.311630+0000","last_active":"2024-07-15T06:50:44.794413+0000","last_peered":"2024-07-15T06:50:44.794413+0000","last_clean":"2024-07-15T06:50:44.794413+0000","last_became_active":"2024-07-15T06:50:29.311134+0000","last_became_peered":"2024-07-15T06:50:29.311134+0000","last_unstale":"2024-07-15T06:50:44.794413+0000","last_undegraded":"2024-07-15T06:50:44.794413+0000","last_fullsized":"2024-07-15T06:50:44.794413+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T16:20:39.971632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.790692+0000","last_change":"2024-07-15T06:50:29.268874+0000","last_active":"2024-07-15T06:50:44.790692+0000","last_peered":"2024-07-15T06:50:44.790692+0000","last_clean":"2024-07-15T06:50:44.790692+0000","last_became_active":"2024-07-15T06:50:29.268302+0000","last_became_peered":"2024-07-15T06:50:29.268302+0000","last_unstale":"2024-07-15T06:50:44.790692+0000","last_undegraded":"2024-07-15T06:50:44.790692+0000","last_fullsized":"2024-07-15T06:50:44.790692+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T16:39:58.460334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":18,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.795107+0000","last_change":"2024-07-15T06:48:28.738368+0000","last_active":"2024-07-15T06:50:44.795107+0000","last_peered":"2024-07-15T06:50:44.795107+0000","last_clean":"2024-07-15T06:50:44.795107+0000","last_became_active":"2024-07-15T06:48:28.737382+0000","last_became_peered":"2024-07-15T06:48:28.737382+0000","last_unstale":"2024-07-15T06:50:44.795107+0000","last_undegraded":"2024-07-15T06:50:44.795107+0000","last_fullsized":"2024-07-15T06:50:44.795107+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_clean_scrub_stamp":"2024-07-15T06:48:27.655327+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T17:01:42.558382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.793570+0000","last_change":"2024-07-15T06:50:29.310565+0000","last_active":"2024-07-15T06:50:44.793570+0000","last_peered":"2024-07-15T06:50:44.793570+0000","last_clean":"2024-07-15T06:50:44.793570+0000","last_became_active":"2024-07-15T06:50:29.310142+0000","last_became_peered":"2024-07-15T06:50:29.310142+0000","last_unstale":"2024-07-15T06:50:44.793570+0000","last_undegraded":"2024-07-15T06:50:44.793570+0000","last_fullsized":"2024-07-15T06:50:44.793570+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T17:53:38.506736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.795518+0000","last_change":"2024-07-15T06:50:29.273932+0000","last_active":"2024-07-15T06:50:44.795518+0000","last_peered":"2024-07-15T06:50:44.795518+0000","last_clean":"2024-07-15T06:50:44.795518+0000","last_became_active":"2024-07-15T06:50:29.273266+0000","last_became_peered":"2024-07-15T06:50:29.273266+0000","last_unstale":"2024-07-15T06:50:44.795518+0000","last_undegraded":"2024-07-15T06:50:44.795518+0000","last_fullsized":"2024-07-15T06:50:44.795518+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T07:02:37.137230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":18,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.804934+0000","last_change":"2024-07-15T06:48:28.872322+0000","last_active":"2024-07-15T06:50:44.804934+0000","last_peered":"2024-07-15T06:50:44.804934+0000","last_clean":"2024-07-15T06:50:44.804934+0000","last_became_active":"2024-07-15T06:48:28.871320+0000","last_became_peered":"2024-07-15T06:48:28.871320+0000","last_unstale":"2024-07-15T06:50:44.804934+0000","last_undegraded":"2024-07-15T06:50:44.804934+0000","last_fullsized":"2024-07-15T06:50:44.804934+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_clean_scrub_stamp":"2024-07-15T06:48:27.655327+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T14:24:12.762681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.789936+0000","last_change":"2024-07-15T06:50:29.846557+0000","last_active":"2024-07-15T06:50:44.789936+0000","last_peered":"2024-07-15T06:50:44.789936+0000","last_clean":"2024-07-15T06:50:44.789936+0000","last_became_active":"2024-07-15T06:50:29.845667+0000","last_became_peered":"2024-07-15T06:50:29.845667+0000","last_unstale":"2024-07-15T06:50:44.789936+0000","last_undegraded":"2024-07-15T06:50:44.789936+0000","last_fullsized":"2024-07-15T06:50:44.789936+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T15:17:50.597165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":17,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.803808+0000","last_change":"2024-07-15T06:50:29.278450+0000","last_active":"2024-07-15T06:50:44.803808+0000","last_peered":"2024-07-15T06:50:44.803808+0000","last_clean":"2024-07-15T06:50:44.803808+0000","last_became_active":"2024-07-15T06:50:29.277938+0000","last_became_peered":"2024-07-15T06:50:29.277938+0000","last_unstale":"2024-07-15T06:50:44.803808+0000","last_undegraded":"2024-07-15T06:50:44.803808+0000","last_fullsized":"2024-07-15T06:50:44.803808+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T15:20:08.735416+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":18,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.794976+0000","last_change":"2024-07-15T06:48:28.872731+0000","last_active":"2024-07-15T06:50:44.794976+0000","last_peered":"2024-07-15T06:50:44.794976+0000","last_clean":"2024-07-15T06:50:44.794976+0000","last_became_active":"2024-07-15T06:48:28.871744+0000","last_became_peered":"2024-07-15T06:48:28.871744+0000","last_unstale":"2024-07-15T06:50:44.794976+0000","last_undegraded":"2024-07-15T06:50:44.794976+0000","last_fullsized":"2024-07-15T06:50:44.794976+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_clean_scrub_stamp":"2024-07-15T06:48:27.655327+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T11:36:09.487214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.801911+0000","last_change":"2024-07-15T06:50:29.848965+0000","last_active":"2024-07-15T06:50:44.801911+0000","last_peered":"2024-07-15T06:50:44.801911+0000","last_clean":"2024-07-15T06:50:44.801911+0000","last_became_active":"2024-07-15T06:50:29.848047+0000","last_became_peered":"2024-07-15T06:50:29.848047+0000","last_unstale":"2024-07-15T06:50:44.801911+0000","last_undegraded":"2024-07-15T06:50:44.801911+0000","last_fullsized":"2024-07-15T06:50:44.801911+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T15:48:12.470869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.790668+0000","last_change":"2024-07-15T06:50:29.263092+0000","last_active":"2024-07-15T06:50:44.790668+0000","last_peered":"2024-07-15T06:50:44.790668+0000","last_clean":"2024-07-15T06:50:44.790668+0000","last_became_active":"2024-07-15T06:50:29.262201+0000","last_became_peered":"2024-07-15T06:50:29.262201+0000","last_unstale":"2024-07-15T06:50:44.790668+0000","last_undegraded":"2024-07-15T06:50:44.790668+0000","last_fullsized":"2024-07-15T06:50:44.790668+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T13:14:27.912283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":18,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.802817+0000","last_change":"2024-07-15T06:48:28.675814+0000","last_active":"2024-07-15T06:50:44.802817+0000","last_peered":"2024-07-15T06:50:44.802817+0000","last_clean":"2024-07-15T06:50:44.802817+0000","last_became_active":"2024-07-15T06:48:28.674798+0000","last_became_peered":"2024-07-15T06:48:28.674798+0000","last_unstale":"2024-07-15T06:50:44.802817+0000","last_undegraded":"2024-07-15T06:50:44.802817+0000","last_fullsized":"2024-07-15T06:50:44.802817+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_clean_scrub_stamp":"2024-07-15T06:48:27.655327+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T14:36:19.382840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.797720+0000","last_change":"2024-07-15T06:50:29.298133+0000","last_active":"2024-07-15T06:50:44.797720+0000","last_peered":"2024-07-15T06:50:44.797720+0000","last_clean":"2024-07-15T06:50:44.797720+0000","last_became_active":"2024-07-15T06:50:29.297265+0000","last_became_peered":"2024-07-15T06:50:29.297265+0000","last_unstale":"2024-07-15T06:50:44.797720+0000","last_undegraded":"2024-07-15T06:50:44.797720+0000","last_fullsized":"2024-07-15T06:50:44.797720+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T08:21:53.743122+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":23,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-07-15T06:50:45.803975+0000","last_change":"2024-07-15T06:50:45.803975+0000","last_active":"2024-07-15T06:50:45.803975+0000","last_peered":"2024-07-15T06:50:45.803975+0000","last_clean":"2024-07-15T06:50:45.803975+0000","last_became_active":"2024-07-15T06:50:45.802915+0000","last_became_peered":"2024-07-15T06:50:45.802915+0000","last_unstale":"2024-07-15T06:50:45.803975+0000","last_undegraded":"2024-07-15T06:50:45.803975+0000","last_fullsized":"2024-07-15T06:50:45.803975+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T15:01:37.417772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"68'7","reported_seq":26,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.798196+0000","last_change":"2024-07-15T06:50:29.256120+0000","last_active":"2024-07-15T06:50:44.798196+0000","last_peered":"2024-07-15T06:50:44.798196+0000","last_clean":"2024-07-15T06:50:44.798196+0000","last_became_active":"2024-07-15T06:50:29.252769+0000","last_became_peered":"2024-07-15T06:50:29.252769+0000","last_unstale":"2024-07-15T06:50:44.798196+0000","last_undegraded":"2024-07-15T06:50:44.798196+0000","last_fullsized":"2024-07-15T06:50:44.798196+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T18:30:32.236566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1552,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":7,"num_read_kb":6,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.793928+0000","last_change":"2024-07-15T06:50:29.311811+0000","last_active":"2024-07-15T06:50:44.793928+0000","last_peered":"2024-07-15T06:50:44.793928+0000","last_clean":"2024-07-15T06:50:44.793928+0000","last_became_active":"2024-07-15T06:50:29.311460+0000","last_became_peered":"2024-07-15T06:50:29.311460+0000","last_unstale":"2024-07-15T06:50:44.793928+0000","last_undegraded":"2024-07-15T06:50:44.793928+0000","last_fullsized":"2024-07-15T06:50:44.793928+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T18:35:18.320094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":19,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.804389+0000","last_change":"2024-07-15T06:50:29.273640+0000","last_active":"2024-07-15T06:50:44.804389+0000","last_peered":"2024-07-15T06:50:44.804389+0000","last_clean":"2024-07-15T06:50:44.804389+0000","last_became_active":"2024-07-15T06:50:29.272992+0000","last_became_peered":"2024-07-15T06:50:29.272992+0000","last_unstale":"2024-07-15T06:50:44.804389+0000","last_undegraded":"2024-07-15T06:50:44.804389+0000","last_fullsized":"2024-07-15T06:50:44.804389+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T08:47:47.418375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":4,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.790601+0000","last_change":"2024-07-15T06:50:29.254597+0000","last_active":"2024-07-15T06:50:44.790601+0000","last_peered":"2024-07-15T06:50:44.790601+0000","last_clean":"2024-07-15T06:50:44.790601+0000","last_became_active":"2024-07-15T06:50:29.253996+0000","last_became_peered":"2024-07-15T06:50:29.253996+0000","last_unstale":"2024-07-15T06:50:44.790601+0000","last_undegraded":"2024-07-15T06:50:44.790601+0000","last_fullsized":"2024-07-15T06:50:44.790601+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T08:04:21.799515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.802216+0000","last_change":"2024-07-15T06:50:29.847314+0000","last_active":"2024-07-15T06:50:44.802216+0000","last_peered":"2024-07-15T06:50:44.802216+0000","last_clean":"2024-07-15T06:50:44.802216+0000","last_became_active":"2024-07-15T06:50:29.846456+0000","last_became_peered":"2024-07-15T06:50:29.846456+0000","last_unstale":"2024-07-15T06:50:44.802216+0000","last_undegraded":"2024-07-15T06:50:44.802216+0000","last_fullsized":"2024-07-15T06:50:44.802216+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T11:42:00.570499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.791328+0000","last_change":"2024-07-15T06:50:29.277238+0000","last_active":"2024-07-15T06:50:44.791328+0000","last_peered":"2024-07-15T06:50:44.791328+0000","last_clean":"2024-07-15T06:50:44.791328+0000","last_became_active":"2024-07-15T06:50:29.276628+0000","last_became_peered":"2024-07-15T06:50:29.276628+0000","last_unstale":"2024-07-15T06:50:44.791328+0000","last_undegraded":"2024-07-15T06:50:44.791328+0000","last_fullsized":"2024-07-15T06:50:44.791328+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T09:10:41.364615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.801890+0000","last_change":"2024-07-15T06:50:29.306634+0000","last_active":"2024-07-15T06:50:44.801890+0000","last_peered":"2024-07-15T06:50:44.801890+0000","last_clean":"2024-07-15T06:50:44.801890+0000","last_became_active":"2024-07-15T06:50:29.306345+0000","last_became_peered":"2024-07-15T06:50:29.306345+0000","last_unstale":"2024-07-15T06:50:44.801890+0000","last_undegraded":"2024-07-15T06:50:44.801890+0000","last_fullsized":"2024-07-15T06:50:44.801890+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T07:14:55.770668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.802585+0000","last_change":"2024-07-15T06:50:29.305890+0000","last_active":"2024-07-15T06:50:44.802585+0000","last_peered":"2024-07-15T06:50:44.802585+0000","last_clean":"2024-07-15T06:50:44.802585+0000","last_became_active":"2024-07-15T06:50:29.305493+0000","last_became_peered":"2024-07-15T06:50:29.305493+0000","last_unstale":"2024-07-15T06:50:44.802585+0000","last_undegraded":"2024-07-15T06:50:44.802585+0000","last_fullsized":"2024-07-15T06:50:44.802585+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T07:40:24.355598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.802157+0000","last_change":"2024-07-15T06:50:29.275180+0000","last_active":"2024-07-15T06:50:44.802157+0000","last_peered":"2024-07-15T06:50:44.802157+0000","last_clean":"2024-07-15T06:50:44.802157+0000","last_became_active":"2024-07-15T06:50:29.274633+0000","last_became_peered":"2024-07-15T06:50:29.274633+0000","last_unstale":"2024-07-15T06:50:44.802157+0000","last_undegraded":"2024-07-15T06:50:44.802157+0000","last_fullsized":"2024-07-15T06:50:44.802157+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T18:15:47.570473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.790235+0000","last_change":"2024-07-15T06:50:29.846346+0000","last_active":"2024-07-15T06:50:44.790235+0000","last_peered":"2024-07-15T06:50:44.790235+0000","last_clean":"2024-07-15T06:50:44.790235+0000","last_became_active":"2024-07-15T06:50:29.845428+0000","last_became_peered":"2024-07-15T06:50:29.845428+0000","last_unstale":"2024-07-15T06:50:44.790235+0000","last_undegraded":"2024-07-15T06:50:44.790235+0000","last_fullsized":"2024-07-15T06:50:44.790235+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T08:10:24.260944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.792398+0000","last_change":"2024-07-15T06:50:29.270851+0000","last_active":"2024-07-15T06:50:44.792398+0000","last_peered":"2024-07-15T06:50:44.792398+0000","last_clean":"2024-07-15T06:50:44.792398+0000","last_became_active":"2024-07-15T06:50:29.270552+0000","last_became_peered":"2024-07-15T06:50:29.270552+0000","last_unstale":"2024-07-15T06:50:44.792398+0000","last_undegraded":"2024-07-15T06:50:44.792398+0000","last_fullsized":"2024-07-15T06:50:44.792398+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T14:41:27.594394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.803449+0000","last_change":"2024-07-15T06:50:29.305640+0000","last_active":"2024-07-15T06:50:44.803449+0000","last_peered":"2024-07-15T06:50:44.803449+0000","last_clean":"2024-07-15T06:50:44.803449+0000","last_became_active":"2024-07-15T06:50:29.305350+0000","last_became_peered":"2024-07-15T06:50:29.305350+0000","last_unstale":"2024-07-15T06:50:44.803449+0000","last_undegraded":"2024-07-15T06:50:44.803449+0000","last_fullsized":"2024-07-15T06:50:44.803449+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T17:18:26.850757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"68'2","reported_seq":16,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.797850+0000","last_change":"2024-07-15T06:50:29.244106+0000","last_active":"2024-07-15T06:50:44.797850+0000","last_peered":"2024-07-15T06:50:44.797850+0000","last_clean":"2024-07-15T06:50:44.797850+0000","last_became_active":"2024-07-15T06:50:29.242294+0000","last_became_peered":"2024-07-15T06:50:29.242294+0000","last_unstale":"2024-07-15T06:50:44.797850+0000","last_undegraded":"2024-07-15T06:50:44.797850+0000","last_fullsized":"2024-07-15T06:50:44.797850+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T15:32:38.649353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":23,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-07-15T06:50:45.912080+0000","last_change":"2024-07-15T06:50:45.912080+0000","last_active":"2024-07-15T06:50:45.912080+0000","last_peered":"2024-07-15T06:50:45.912080+0000","last_clean":"2024-07-15T06:50:45.912080+0000","last_became_active":"2024-07-15T06:50:45.911055+0000","last_became_peered":"2024-07-15T06:50:45.911055+0000","last_unstale":"2024-07-15T06:50:45.912080+0000","last_undegraded":"2024-07-15T06:50:45.912080+0000","last_fullsized":"2024-07-15T06:50:45.912080+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T17:41:11.718209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"68'2","reported_seq":16,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.790341+0000","last_change":"2024-07-15T06:50:29.266698+0000","last_active":"2024-07-15T06:50:44.790341+0000","last_peered":"2024-07-15T06:50:44.790341+0000","last_clean":"2024-07-15T06:50:44.790341+0000","last_became_active":"2024-07-15T06:50:29.266355+0000","last_became_peered":"2024-07-15T06:50:29.266355+0000","last_unstale":"2024-07-15T06:50:44.790341+0000","last_undegraded":"2024-07-15T06:50:44.790341+0000","last_fullsized":"2024-07-15T06:50:44.790341+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T09:58:47.439567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.798794+0000","last_change":"2024-07-15T06:50:29.303449+0000","last_active":"2024-07-15T06:50:44.798794+0000","last_peered":"2024-07-15T06:50:44.798794+0000","last_clean":"2024-07-15T06:50:44.798794+0000","last_became_active":"2024-07-15T06:50:29.303028+0000","last_became_peered":"2024-07-15T06:50:29.303028+0000","last_unstale":"2024-07-15T06:50:44.798794+0000","last_undegraded":"2024-07-15T06:50:44.798794+0000","last_fullsized":"2024-07-15T06:50:44.798794+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T16:33:21.086414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":17,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.790394+0000","last_change":"2024-07-15T06:50:29.268693+0000","last_active":"2024-07-15T06:50:44.790394+0000","last_peered":"2024-07-15T06:50:44.790394+0000","last_clean":"2024-07-15T06:50:44.790394+0000","last_became_active":"2024-07-15T06:50:29.267932+0000","last_became_peered":"2024-07-15T06:50:29.267932+0000","last_unstale":"2024-07-15T06:50:44.790394+0000","last_undegraded":"2024-07-15T06:50:44.790394+0000","last_fullsized":"2024-07-15T06:50:44.790394+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T14:55:15.826736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.797649+0000","last_change":"2024-07-15T06:50:29.303795+0000","last_active":"2024-07-15T06:50:44.797649+0000","last_peered":"2024-07-15T06:50:44.797649+0000","last_clean":"2024-07-15T06:50:44.797649+0000","last_became_active":"2024-07-15T06:50:29.303252+0000","last_became_peered":"2024-07-15T06:50:29.303252+0000","last_unstale":"2024-07-15T06:50:44.797649+0000","last_undegraded":"2024-07-15T06:50:44.797649+0000","last_fullsized":"2024-07-15T06:50:44.797649+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T12:52:31.000663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":17,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.790584+0000","last_change":"2024-07-15T06:50:29.255053+0000","last_active":"2024-07-15T06:50:44.790584+0000","last_peered":"2024-07-15T06:50:44.790584+0000","last_clean":"2024-07-15T06:50:44.790584+0000","last_became_active":"2024-07-15T06:50:29.253310+0000","last_became_peered":"2024-07-15T06:50:29.253310+0000","last_unstale":"2024-07-15T06:50:44.790584+0000","last_undegraded":"2024-07-15T06:50:44.790584+0000","last_fullsized":"2024-07-15T06:50:44.790584+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T08:09:48.203498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.802395+0000","last_change":"2024-07-15T06:50:29.302910+0000","last_active":"2024-07-15T06:50:44.802395+0000","last_peered":"2024-07-15T06:50:44.802395+0000","last_clean":"2024-07-15T06:50:44.802395+0000","last_became_active":"2024-07-15T06:50:29.301648+0000","last_became_peered":"2024-07-15T06:50:29.301648+0000","last_unstale":"2024-07-15T06:50:44.802395+0000","last_undegraded":"2024-07-15T06:50:44.802395+0000","last_fullsized":"2024-07-15T06:50:44.802395+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T12:33:30.880775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'2","reported_seq":16,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.803561+0000","last_change":"2024-07-15T06:50:29.271689+0000","last_active":"2024-07-15T06:50:44.803561+0000","last_peered":"2024-07-15T06:50:44.803561+0000","last_clean":"2024-07-15T06:50:44.803561+0000","last_became_active":"2024-07-15T06:50:29.271387+0000","last_became_peered":"2024-07-15T06:50:29.271387+0000","last_unstale":"2024-07-15T06:50:44.803561+0000","last_undegraded":"2024-07-15T06:50:44.803561+0000","last_fullsized":"2024-07-15T06:50:44.803561+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T13:06:39.901886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.797211+0000","last_change":"2024-07-15T06:50:29.301207+0000","last_active":"2024-07-15T06:50:44.797211+0000","last_peered":"2024-07-15T06:50:44.797211+0000","last_clean":"2024-07-15T06:50:44.797211+0000","last_became_active":"2024-07-15T06:50:29.300787+0000","last_became_peered":"2024-07-15T06:50:29.300787+0000","last_unstale":"2024-07-15T06:50:44.797211+0000","last_undegraded":"2024-07-15T06:50:44.797211+0000","last_fullsized":"2024-07-15T06:50:44.797211+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T07:47:03.031066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"68'5","reported_seq":21,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.791179+0000","last_change":"2024-07-15T06:50:29.252625+0000","last_active":"2024-07-15T06:50:44.791179+0000","last_peered":"2024-07-15T06:50:44.791179+0000","last_clean":"2024-07-15T06:50:44.791179+0000","last_became_active":"2024-07-15T06:50:29.252276+0000","last_became_peered":"2024-07-15T06:50:29.252276+0000","last_unstale":"2024-07-15T06:50:44.791179+0000","last_undegraded":"2024-07-15T06:50:44.791179+0000","last_fullsized":"2024-07-15T06:50:44.791179+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T14:34:20.688321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":4,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2338,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":47,"num_read_kb":49,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":7014,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":45,"ondisk_log_size":45,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234053,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298664,"kb_used_data":1304,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446488,"statfs":{"total":95995035648,"available":95689203712,"internally_reserved":0,"allocated":1335296,"data_stored":905399,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Jul 15 06:51:55 2024","interfaces":[{"interface":"back","average":{"1min":0.94699999999999995,"5min":0.93600000000000005,"15min":0.93700000000000006},"min":{"1min":0.56699999999999995,"5min":0.443,"15min":0.443},"max":{"1min":1.1919999999999999,"5min":1.532,"15min":1.532},"last":0.63100000000000001},{"interface":"front","average":{"1min":0.85199999999999998,"5min":0.88400000000000001,"15min":0.89500000000000002},"min":{"1min":0.57299999999999995,"5min":0.46600000000000003,"15min":0.46600000000000003},"max":{"1min":1.1040000000000001,"5min":2.073,"15min":2.073},"last":0.81599999999999995}]},{"osd":1,"last update":"Mon Jul 15 06:51:55 2024","interfaces":[{"interface":"back","average":{"1min":0.90200000000000002,"5min":0.90400000000000003,"15min":0.93000000000000005},"min":{"1min":0.58699999999999997,"5min":0.48499999999999999,"15min":0.48499999999999999},"max":{"1min":1.1930000000000001,"5min":1.6579999999999999,"15min":1.6579999999999999},"last":0.58299999999999996},{"interface":"front","average":{"1min":0.93400000000000005,"5min":0.95599999999999996,"15min":0.99399999999999999},"min":{"1min":0.69199999999999995,"5min":0.51500000000000001,"15min":0.51500000000000001},"max":{"1min":1.1279999999999999,"5min":2.1840000000000002,"15min":2.1840000000000002},"last":0.67500000000000004}]},{"osd":2,"last update":"Mon Jul 15 06:51:55 2024","interfaces":[{"interface":"back","average":{"1min":0.873,"5min":0.91700000000000004,"15min":0.90800000000000003},"min":{"1min":0.66300000000000003,"5min":0.50900000000000001,"15min":0.50900000000000001},"max":{"1min":1.119,"5min":1.6279999999999999,"15min":1.6279999999999999},"last":0.77900000000000003},{"interface":"front","average":{"1min":1.006,"5min":0.96499999999999997,"15min":0.97899999999999998},"min":{"1min":0.64100000000000001,"5min":0.55800000000000005,"15min":0.55800000000000005},"max":{"1min":1.2030000000000001,"5min":1.802,"15min":1.802},"last":0.93899999999999995}]},{"osd":3,"last update":"Mon Jul 15 06:51:55 2024","interfaces":[{"interface":"back","average":{"1min":0.872,"5min":0.93300000000000005,"15min":0.93999999999999995},"min":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"max":{"1min":1.2310000000000001,"5min":1.704,"15min":1.704},"last":0.70899999999999996},{"interface":"front","average":{"1min":1.0189999999999999,"5min":1.018,"15min":1.054},"min":{"1min":0.60599999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"max":{"1min":1.28,"5min":1.74,"15min":1.74},"last":1.0589999999999999}]},{"osd":4,"last update":"Mon Jul 15 06:51:55 2024","interfaces":[{"interface":"back","average":{"1min":0.93600000000000005,"5min":0.95099999999999996,"15min":0.96699999999999997},"min":{"1min":0.69799999999999995,"5min":0.52500000000000002,"15min":0.52500000000000002},"max":{"1min":1.306,"5min":2.395,"15min":2.395},"last":0.88300000000000001},{"interface":"front","average":{"1min":1.0629999999999999,"5min":1.0249999999999999,"15min":1.0569999999999999},"min":{"1min":0.81699999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"max":{"1min":1.2729999999999999,"5min":2.129,"15min":2.129},"last":0.84399999999999997}]},{"osd":5,"last update":"Mon Jul 15 06:51:55 2024","interfaces":[{"interface":"back","average":{"1min":1.014,"5min":0.996,"15min":1.0389999999999999},"min":{"1min":0.65500000000000003,"5min":0.47999999999999998,"15min":0.47999999999999998},"max":{"1min":1.2889999999999999,"5min":2.7759999999999998,"15min":2.7759999999999998},"last":1.1319999999999999},{"interface":"front","average":{"1min":1.0660000000000001,"5min":1.0489999999999999,"15min":1.075},"min":{"1min":0.89300000000000002,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":1.3160000000000001,"5min":2.254,"15min":2.254},"last":1.0429999999999999}]},{"osd":6,"last update":"Mon Jul 15 06:51:55 2024","interfaces":[{"interface":"back","average":{"1min":1.109,"5min":1.0860000000000001,"15min":1.1200000000000001},"min":{"1min":0.94799999999999995,"5min":0.50600000000000001,"15min":0.50600000000000001},"max":{"1min":1.2709999999999999,"5min":2.3260000000000001,"15min":2.3260000000000001},"last":0.98999999999999999},{"interface":"front","average":{"1min":1.089,"5min":1.099,"15min":1.127},"min":{"1min":0.73599999999999999,"5min":0.59499999999999997,"15min":0.59499999999999997},"max":{"1min":1.292,"5min":1.893,"15min":1.893},"last":0.90300000000000002}]}]},{"osd":6,"up_from":46,"seq":197568495695,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298656,"kb_used_data":1296,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446496,"statfs":{"total":95995035648,"available":95689211904,"internally_reserved":0,"allocated":1327104,"data_stored":905576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Jul 15 06:51:58 2024","interfaces":[{"interface":"back","average":{"1min":0.871,"5min":0.85299999999999998,"15min":0.90600000000000003},"min":{"1min":0.53300000000000003,"5min":0.50700000000000001,"15min":0.50700000000000001},"max":{"1min":1.1679999999999999,"5min":1.268,"15min":2.1579999999999999},"last":0.93400000000000005},{"interface":"front","average":{"1min":0.90800000000000003,"5min":0.879,"15min":0.89100000000000001},"min":{"1min":0.56200000000000006,"5min":0.47099999999999997,"15min":0.47099999999999997},"max":{"1min":1.165,"5min":1.276,"15min":1.3089999999999999},"last":1.0049999999999999}]},{"osd":1,"last update":"Mon Jul 15 06:51:58 2024","interfaces":[{"interface":"back","average":{"1min":0.92400000000000004,"5min":0.90800000000000003,"15min":0.91200000000000003},"min":{"1min":0.55700000000000005,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":1.1080000000000001,"5min":1.4019999999999999,"15min":1.444},"last":1.1859999999999999},{"interface":"front","average":{"1min":0.94499999999999995,"5min":0.88700000000000001,"15min":0.94899999999999995},"min":{"1min":0.623,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":1.1930000000000001,"5min":1.4119999999999999,"15min":2.2210000000000001},"last":0.79100000000000004}]},{"osd":2,"last update":"Mon Jul 15 06:51:58 2024","interfaces":[{"interface":"back","average":{"1min":0.84399999999999997,"5min":0.86299999999999999,"15min":0.95999999999999996},"min":{"1min":0.66300000000000003,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":1.327,"5min":1.327,"15min":2.754},"last":1.0509999999999999},{"interface":"front","average":{"1min":0.995,"5min":0.93000000000000005,"15min":0.99399999999999999},"min":{"1min":0.65800000000000003,"5min":0.501,"15min":0.501},"max":{"1min":1.28,"5min":1.536,"15min":2.1019999999999999},"last":1.0649999999999999}]},{"osd":3,"last update":"Mon Jul 15 06:51:58 2024","interfaces":[{"interface":"back","average":{"1min":0.97099999999999997,"5min":0.93300000000000005,"15min":0.91300000000000003},"min":{"1min":0.63300000000000001,"5min":0.498,"15min":0.498},"max":{"1min":1.198,"5min":1.3799999999999999,"15min":1.3799999999999999},"last":1.0329999999999999},{"interface":"front","average":{"1min":0.97899999999999998,"5min":0.93600000000000005,"15min":0.99199999999999999},"min":{"1min":0.69099999999999995,"5min":0.52500000000000002,"15min":0.52500000000000002},"max":{"1min":1.202,"5min":1.321,"15min":2.0600000000000001},"last":1.143}]},{"osd":4,"last update":"Mon Jul 15 06:51:58 2024","interfaces":[{"interface":"back","average":{"1min":0.84399999999999997,"5min":0.82899999999999996,"15min":0.79800000000000004},"min":{"1min":0.59599999999999997,"5min":0.38700000000000001,"15min":0.34799999999999998},"max":{"1min":1.181,"5min":1.6120000000000001,"15min":1.6120000000000001},"last":0.88600000000000001},{"interface":"front","average":{"1min":0.97099999999999997,"5min":0.93200000000000005,"15min":0.872},"min":{"1min":0.66500000000000004,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":1.264,"5min":1.417,"15min":1.417},"last":0.97399999999999998}]},{"osd":5,"last update":"Mon Jul 15 06:51:58 2024","interfaces":[{"interface":"back","average":{"1min":0.93899999999999995,"5min":0.91800000000000004,"15min":0.878},"min":{"1min":0.64800000000000002,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":1.256,"5min":1.6779999999999999,"15min":1.6779999999999999},"last":1.121},{"interface":"front","average":{"1min":0.97799999999999998,"5min":0.98599999999999999,"15min":0.89500000000000002},"min":{"1min":0.62,"5min":0.373,"15min":0.373},"max":{"1min":1.3160000000000001,"5min":1.375,"15min":1.375},"last":0.83899999999999997}]},{"osd":7,"last update":"Mon Jul 15 06:51:56 2024","interfaces":[{"interface":"back","average":{"1min":1.032,"5min":1.02,"15min":0.996},"min":{"1min":0.68799999999999994,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":1.3979999999999999,"5min":1.7430000000000001,"15min":1.7430000000000001},"last":1.214},{"interface":"front","average":{"1min":1.1140000000000001,"5min":1.0349999999999999,"15min":1.0049999999999999},"min":{"1min":0.75600000000000001,"5min":0.435,"15min":0.435},"max":{"1min":1.3680000000000001,"5min":1.8540000000000001,"15min":1.8540000000000001},"last":1.0940000000000001}]}]},{"osd":5,"up_from":38,"seq":163208757337,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298064,"kb_used_data":704,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447088,"statfs":{"total":95995035648,"available":95689818112,"internally_reserved":0,"allocated":720896,"data_stored":313903,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Jul 15 06:52:10 2024","interfaces":[{"interface":"back","average":{"1min":0.99399999999999999,"5min":0.93899999999999995,"15min":0.93000000000000005},"min":{"1min":0.59099999999999997,"5min":0.47199999999999998,"15min":0.47199999999999998},"max":{"1min":1.2,"5min":1.242,"15min":1.242},"last":1.0700000000000001},{"interface":"front","average":{"1min":0.99199999999999999,"5min":0.94799999999999995,"15min":0.91100000000000003},"min":{"1min":0.70299999999999996,"5min":0.502,"15min":0.502},"max":{"1min":1.2010000000000001,"5min":1.27,"15min":1.27},"last":1.097}]},{"osd":1,"last update":"Mon Jul 15 06:52:10 2024","interfaces":[{"interface":"back","average":{"1min":0.91900000000000004,"5min":0.88400000000000001,"15min":0.83199999999999996},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":1.2010000000000001,"5min":1.2110000000000001,"15min":1.2110000000000001},"last":0.86599999999999999},{"interface":"front","average":{"1min":0.86099999999999999,"5min":0.89100000000000001,"15min":0.88700000000000001},"min":{"1min":0.65000000000000002,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":1.206,"5min":1.268,"15min":1.268},"last":0.96999999999999997}]},{"osd":2,"last update":"Mon Jul 15 06:52:10 2024","interfaces":[{"interface":"back","average":{"1min":0.877,"5min":0.873,"15min":0.83099999999999996},"min":{"1min":0.59399999999999997,"5min":0.52500000000000002,"15min":0.52500000000000002},"max":{"1min":1.194,"5min":1.2869999999999999,"15min":1.2869999999999999},"last":0.78900000000000003},{"interface":"front","average":{"1min":0.90600000000000003,"5min":0.88200000000000001,"15min":0.91000000000000003},"min":{"1min":0.55100000000000005,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":1.2709999999999999,"5min":1.2709999999999999,"15min":1.286},"last":1.2709999999999999}]},{"osd":3,"last update":"Mon Jul 15 06:52:10 2024","interfaces":[{"interface":"back","average":{"1min":0.93300000000000005,"5min":0.92700000000000005,"15min":0.86899999999999999},"min":{"1min":0.73599999999999999,"5min":0.56399999999999995,"15min":0.56399999999999995},"max":{"1min":1.2,"5min":1.2709999999999999,"15min":1.2709999999999999},"last":1.0409999999999999},{"interface":"front","average":{"1min":0.96099999999999997,"5min":0.93600000000000005,"15min":0.94399999999999995},"min":{"1min":0.70599999999999996,"5min":0.53500000000000003,"15min":0.53500000000000003},"max":{"1min":1.2310000000000001,"5min":1.256,"15min":1.3320000000000001},"last":1.01}]},{"osd":4,"last update":"Mon Jul 15 06:52:10 2024","interfaces":[{"interface":"back","average":{"1min":0.871,"5min":0.81399999999999995,"15min":0.752},"min":{"1min":0.63,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":1.0760000000000001,"5min":1.1910000000000001,"15min":1.1910000000000001},"last":0.90000000000000002},{"interface":"front","average":{"1min":0.93100000000000005,"5min":0.88300000000000001,"15min":0.79400000000000004},"min":{"1min":0.72199999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":1.1779999999999999,"5min":1.3420000000000001,"15min":1.3420000000000001},"last":0.93200000000000005}]},{"osd":6,"last update":"Mon Jul 15 06:52:04 2024","interfaces":[{"interface":"back","average":{"1min":0.96899999999999997,"5min":0.93600000000000005,"15min":0.91000000000000003},"min":{"1min":0.61599999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":1.333,"5min":1.333,"15min":1.333},"last":1.2010000000000001},{"interface":"front","average":{"1min":0.98199999999999998,"5min":0.96999999999999997,"15min":0.93300000000000005},"min":{"1min":0.63100000000000001,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":1.2869999999999999,"5min":1.3149999999999999,"15min":1.3149999999999999},"last":1.175}]},{"osd":7,"last update":"Mon Jul 15 06:51:50 2024","interfaces":[{"interface":"back","average":{"1min":1.046,"5min":1.0389999999999999,"15min":0.99299999999999999},"min":{"1min":0.77800000000000002,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":1.3600000000000001,"5min":1.3600000000000001,"15min":1.3600000000000001},"last":1.236},{"interface":"front","average":{"1min":1.046,"5min":1.026,"15min":0.998},"min":{"1min":0.85699999999999998,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":1.27,"5min":1.3939999999999999,"15min":1.3939999999999999},"last":1.1399999999999999}]}]},{"osd":4,"up_from":30,"seq":128849018980,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298084,"kb_used_data":724,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447068,"statfs":{"total":95995035648,"available":95689797632,"internally_reserved":0,"allocated":741376,"data_stored":315284,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Jul 15 06:51:15 2024","interfaces":[{"interface":"back","average":{"1min":0.82599999999999996,"5min":0.85099999999999998,"15min":0.83099999999999996},"min":{"1min":0.52400000000000002,"5min":0.46600000000000003,"15min":0.46600000000000003},"max":{"1min":1.252,"5min":1.381,"15min":1.381},"last":0.871},{"interface":"front","average":{"1min":0.98999999999999999,"5min":0.91700000000000004,"15min":0.84699999999999998},"min":{"1min":0.70699999999999996,"5min":0.48799999999999999,"15min":0.48799999999999999},"max":{"1min":1.2589999999999999,"5min":1.381,"15min":1.381},"last":0.78600000000000003}]},{"osd":1,"last update":"Mon Jul 15 06:51:15 2024","interfaces":[{"interface":"back","average":{"1min":0.98499999999999999,"5min":0.93200000000000005,"15min":0.878},"min":{"1min":0.60999999999999999,"5min":0.497,"15min":0.497},"max":{"1min":1.24,"5min":1.3120000000000001,"15min":1.3120000000000001},"last":1.1479999999999999},{"interface":"front","average":{"1min":0.95499999999999996,"5min":0.85899999999999999,"15min":0.86899999999999999},"min":{"1min":0.56799999999999995,"5min":0.501,"15min":0.501},"max":{"1min":1.2609999999999999,"5min":1.3260000000000001,"15min":1.3260000000000001},"last":1.0089999999999999}]},{"osd":2,"last update":"Mon Jul 15 06:51:15 2024","interfaces":[{"interface":"back","average":{"1min":0.94799999999999995,"5min":0.90600000000000003,"15min":0.84899999999999998},"min":{"1min":0.70699999999999996,"5min":0.49399999999999999,"15min":0.49399999999999999},"max":{"1min":1.333,"5min":1.333,"15min":1.333},"last":1.0900000000000001},{"interface":"front","average":{"1min":0.87,"5min":0.86599999999999999,"15min":0.873},"min":{"1min":0.55300000000000005,"5min":0.52100000000000002,"15min":0.52100000000000002},"max":{"1min":1.2430000000000001,"5min":1.9910000000000001,"15min":1.9910000000000001},"last":1.1220000000000001}]},{"osd":3,"last update":"Mon Jul 15 06:51:15 2024","interfaces":[{"interface":"back","average":{"1min":0.92900000000000005,"5min":0.91100000000000003,"15min":0.874},"min":{"1min":0.59199999999999997,"5min":0.56000000000000005,"15min":0.56000000000000005},"max":{"1min":1.1579999999999999,"5min":2.1709999999999998,"15min":2.1709999999999998},"last":0.96499999999999997},{"interface":"front","average":{"1min":1.0069999999999999,"5min":0.96199999999999997,"15min":0.91600000000000004},"min":{"1min":0.66100000000000003,"5min":0.48999999999999999,"15min":0.48999999999999999},"max":{"1min":1.264,"5min":1.4970000000000001,"15min":1.4970000000000001},"last":1.0620000000000001}]},{"osd":5,"last update":"Mon Jul 15 06:51:11 2024","interfaces":[{"interface":"back","average":{"1min":0.88200000000000001,"5min":0.86699999999999999,"15min":0.75600000000000001},"min":{"1min":0.441,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":1.1060000000000001,"5min":2.085,"15min":2.085},"last":0.82699999999999996},{"interface":"front","average":{"1min":0.92400000000000004,"5min":0.85199999999999998,"15min":0.748},"min":{"1min":0.42499999999999999,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":1.254,"5min":1.2729999999999999,"15min":1.2729999999999999},"last":0.751}]},{"osd":6,"last update":"Mon Jul 15 06:52:01 2024","interfaces":[{"interface":"back","average":{"1min":0.98099999999999998,"5min":0.95099999999999996,"15min":0.79300000000000004},"min":{"1min":0.65200000000000002,"5min":0.35399999999999998,"15min":0.35099999999999998},"max":{"1min":1.3180000000000001,"5min":1.5760000000000001,"15min":1.5760000000000001},"last":0.66500000000000004},{"interface":"front","average":{"1min":1.0169999999999999,"5min":0.96999999999999997,"15min":0.82199999999999995},"min":{"1min":0.749,"5min":0.36799999999999999,"15min":0.34000000000000002},"max":{"1min":1.2330000000000001,"5min":1.347,"15min":1.347},"last":0.91000000000000003}]},{"osd":7,"last update":"Mon Jul 15 06:51:53 2024","interfaces":[{"interface":"back","average":{"1min":1.036,"5min":1.0049999999999999,"15min":0.94499999999999995},"min":{"1min":0.80200000000000005,"5min":0.378,"15min":0.378},"max":{"1min":1.264,"5min":2.2469999999999999,"15min":2.2469999999999999},"last":1.032},{"interface":"front","average":{"1min":1.107,"5min":1.071,"15min":1.0649999999999999},"min":{"1min":0.749,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":1.3700000000000001,"5min":1.653,"15min":1.653},"last":0.996}]}]},{"osd":3,"up_from":25,"seq":107374182511,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298060,"kb_used_data":704,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447092,"statfs":{"total":95995035648,"available":95689822208,"internally_reserved":0,"allocated":720896,"data_stored":313676,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Jul 15 06:51:24 2024","interfaces":[{"interface":"back","average":{"1min":0.64500000000000002,"5min":0.65700000000000003,"15min":0.64400000000000002},"min":{"1min":0.45600000000000002,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.90700000000000003,"5min":1.1220000000000001,"15min":1.1220000000000001},"last":0.53700000000000003},{"interface":"front","average":{"1min":0.76600000000000001,"5min":0.72299999999999998,"15min":0.70399999999999996},"min":{"1min":0.47199999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":1.056,"5min":1.647,"15min":1.647},"last":0.77000000000000002}]},{"osd":1,"last update":"Mon Jul 15 06:51:24 2024","interfaces":[{"interface":"back","average":{"1min":0.68100000000000005,"5min":0.72399999999999998,"15min":0.70699999999999996},"min":{"1min":0.54000000000000004,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.83999999999999997,"5min":1.3580000000000001,"15min":1.3580000000000001},"last":0.622},{"interface":"front","average":{"1min":0.70299999999999996,"5min":0.72099999999999997,"15min":0.69999999999999996},"min":{"1min":0.49399999999999999,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.998,"5min":1.149,"15min":1.2190000000000001},"last":0.85199999999999998}]},{"osd":2,"last update":"Mon Jul 15 06:51:24 2024","interfaces":[{"interface":"back","average":{"1min":0.84399999999999997,"5min":0.79000000000000004,"15min":0.76200000000000001},"min":{"1min":0.56399999999999995,"5min":0.374,"15min":0.374},"max":{"1min":1.2330000000000001,"5min":1.2330000000000001,"15min":1.2330000000000001},"last":0.94199999999999995},{"interface":"front","average":{"1min":0.77900000000000003,"5min":0.752,"15min":0.72699999999999998},"min":{"1min":0.59899999999999998,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":1.0369999999999999,"5min":1.3240000000000001,"15min":1.3240000000000001},"last":0.72099999999999997}]},{"osd":4,"last update":"Mon Jul 15 06:51:18 2024","interfaces":[{"interface":"back","average":{"1min":1.006,"5min":0.94599999999999995,"15min":0.89000000000000001},"min":{"1min":0.70099999999999996,"5min":0.59599999999999997,"15min":0.59599999999999997},"max":{"1min":1.198,"5min":1.2949999999999999,"15min":1.2949999999999999},"last":0.82199999999999995},{"interface":"front","average":{"1min":0.97099999999999997,"5min":0.95799999999999996,"15min":0.91200000000000003},"min":{"1min":0.76900000000000002,"5min":0.57299999999999995,"15min":0.57299999999999995},"max":{"1min":1.1719999999999999,"5min":1.8140000000000001,"15min":1.8140000000000001},"last":1.048}]},{"osd":5,"last update":"Mon Jul 15 06:51:15 2024","interfaces":[{"interface":"back","average":{"1min":1.0369999999999999,"5min":0.995,"15min":0.96999999999999997},"min":{"1min":0.79900000000000004,"5min":0.57299999999999995,"15min":0.57299999999999995},"max":{"1min":1.3500000000000001,"5min":1.7609999999999999,"15min":1.7609999999999999},"last":0.91800000000000004},{"interface":"front","average":{"1min":0.97499999999999998,"5min":0.95899999999999996,"15min":0.97899999999999998},"min":{"1min":0.81599999999999995,"5min":0.64900000000000002,"15min":0.64900000000000002},"max":{"1min":1.1439999999999999,"5min":1.429,"15min":1.5069999999999999},"last":0.98699999999999999}]},{"osd":6,"last update":"Mon Jul 15 06:52:07 2024","interfaces":[{"interface":"back","average":{"1min":1.042,"5min":1.018,"15min":1.069},"min":{"1min":0.66100000000000003,"5min":0.65600000000000003,"15min":0.64500000000000002},"max":{"1min":1.542,"5min":1.5880000000000001,"15min":1.8899999999999999},"last":1.133},{"interface":"front","average":{"1min":1.0640000000000001,"5min":0.996,"15min":1.0129999999999999},"min":{"1min":0.82999999999999996,"5min":0.61499999999999999,"15min":0.61499999999999999},"max":{"1min":1.6120000000000001,"5min":1.6120000000000001,"15min":1.6120000000000001},"last":1.0209999999999999}]},{"osd":7,"last update":"Mon Jul 15 06:51:57 2024","interfaces":[{"interface":"back","average":{"1min":1.206,"5min":1.054,"15min":1.034},"min":{"1min":0.80900000000000005,"5min":0.58299999999999996,"15min":0.58299999999999996},"max":{"1min":2.6960000000000002,"5min":2.6960000000000002,"15min":2.6960000000000002},"last":1.0900000000000001},{"interface":"front","average":{"1min":1.103,"5min":1.056,"15min":1.0900000000000001},"min":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"max":{"1min":1.7509999999999999,"5min":1.7509999999999999,"15min":1.7509999999999999},"last":0.88900000000000001}]}]},{"osd":2,"up_from":18,"seq":77309411448,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298064,"kb_used_data":708,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447088,"statfs":{"total":95995035648,"available":95689818112,"internally_reserved":0,"allocated":724992,"data_stored":314190,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Jul 15 06:51:42 2024","interfaces":[{"interface":"back","average":{"1min":0.58499999999999996,"5min":0.623,"15min":0.59599999999999997},"min":{"1min":0.30399999999999999,"5min":0.28299999999999997,"15min":0.22700000000000001},"max":{"1min":0.77300000000000002,"5min":1.4690000000000001,"15min":1.4690000000000001},"last":0.61099999999999999},{"interface":"front","average":{"1min":0.68100000000000005,"5min":0.68000000000000005,"15min":0.61699999999999999},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":1.0840000000000001,"5min":1.0880000000000001,"15min":1.095},"last":0.94399999999999995}]},{"osd":1,"last update":"Mon Jul 15 06:51:42 2024","interfaces":[{"interface":"back","average":{"1min":0.66300000000000003,"5min":0.70699999999999996,"15min":0.65900000000000003},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":1.0940000000000001,"5min":1.3149999999999999,"15min":1.3149999999999999},"last":0.78300000000000003},{"interface":"front","average":{"1min":0.753,"5min":0.75700000000000001,"15min":0.67700000000000005},"min":{"1min":0.215,"5min":0.215,"15min":0.215},"max":{"1min":1.1779999999999999,"5min":1.2330000000000001,"15min":1.2330000000000001},"last":0.65900000000000003}]},{"osd":3,"last update":"Mon Jul 15 06:51:31 2024","interfaces":[{"interface":"back","average":{"1min":0.76000000000000001,"5min":0.76100000000000001,"15min":0.75700000000000001},"min":{"1min":0.35699999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":1.0600000000000001,"5min":1.256,"15min":1.256},"last":0.91000000000000003},{"interface":"front","average":{"1min":0.80700000000000005,"5min":0.82199999999999995,"15min":0.76000000000000001},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":1.133,"5min":1.264,"15min":1.264},"last":0.86099999999999999}]},{"osd":4,"last update":"Mon Jul 15 06:51:14 2024","interfaces":[{"interface":"back","average":{"1min":1.077,"5min":0.98899999999999999,"15min":0.91400000000000003},"min":{"1min":0.83699999999999997,"5min":0.56299999999999994,"15min":0.56299999999999994},"max":{"1min":1.5129999999999999,"5min":1.5129999999999999,"15min":1.5129999999999999},"last":1.159},{"interface":"front","average":{"1min":1.121,"5min":1.0569999999999999,"15min":0.97099999999999997},"min":{"1min":0.89700000000000002,"5min":0.50800000000000001,"15min":0.50800000000000001},"max":{"1min":1.3520000000000001,"5min":1.5589999999999999,"15min":1.5589999999999999},"last":1.137}]},{"osd":5,"last update":"Mon Jul 15 06:51:14 2024","interfaces":[{"interface":"back","average":{"1min":1.0409999999999999,"5min":1.0229999999999999,"15min":0.97799999999999998},"min":{"1min":0.72799999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"max":{"1min":1.444,"5min":1.7290000000000001,"15min":1.7290000000000001},"last":1.1739999999999999},{"interface":"front","average":{"1min":1.028,"5min":0.98099999999999998,"15min":0.96399999999999997},"min":{"1min":0.70199999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"max":{"1min":1.401,"5min":1.571,"15min":1.571},"last":1.075}]},{"osd":6,"last update":"Mon Jul 15 06:52:06 2024","interfaces":[{"interface":"back","average":{"1min":0.98999999999999999,"5min":1.038,"15min":1.0600000000000001},"min":{"1min":0.53300000000000003,"5min":0.52800000000000002,"15min":0.52800000000000002},"max":{"1min":1.349,"5min":1.8009999999999999,"15min":1.8009999999999999},"last":1.052},{"interface":"front","average":{"1min":0.91000000000000003,"5min":0.94199999999999995,"15min":0.95099999999999996},"min":{"1min":0.64000000000000001,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":1.141,"5min":2.3500000000000001,"15min":2.3500000000000001},"last":1.0209999999999999}]},{"osd":7,"last update":"Mon Jul 15 06:51:54 2024","interfaces":[{"interface":"back","average":{"1min":1.02,"5min":1.038,"15min":1.0589999999999999},"min":{"1min":0.67600000000000005,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":1.468,"5min":1.6299999999999999,"15min":1.6299999999999999},"last":1.1200000000000001},{"interface":"front","average":{"1min":1.022,"5min":1.087,"15min":1.075},"min":{"1min":0.63800000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"max":{"1min":1.5469999999999999,"5min":1.756,"15min":1.756},"last":1.097}]}]},{"osd":1,"up_from":13,"seq":55834574979,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298056,"kb_used_data":700,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447096,"statfs":{"total":95995035648,"available":95689826304,"internally_reserved":0,"allocated":716800,"data_stored":313586,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Jul 15 06:51:49 2024","interfaces":[{"interface":"back","average":{"1min":0.63600000000000001,"5min":0.65400000000000003,"15min":0.59099999999999997},"min":{"1min":0.38400000000000001,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.96899999999999997,"5min":1.2250000000000001,"15min":1.2250000000000001},"last":0.61599999999999999},{"interface":"front","average":{"1min":0.70699999999999996,"5min":0.71599999999999997,"15min":0.63100000000000001},"min":{"1min":0.39600000000000002,"5min":0.32700000000000001,"15min":0.312},"max":{"1min":1.1479999999999999,"5min":1.2709999999999999,"15min":1.5609999999999999},"last":0.73199999999999998}]},{"osd":2,"last update":"Mon Jul 15 06:51:30 2024","interfaces":[{"interface":"back","average":{"1min":0.68400000000000005,"5min":0.749,"15min":0.68600000000000005},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.28299999999999997},"max":{"1min":1.1339999999999999,"5min":1.4379999999999999,"15min":1.7509999999999999},"last":0.85499999999999998},{"interface":"front","average":{"1min":0.71999999999999997,"5min":0.74299999999999999,"15min":0.64900000000000002},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.29599999999999999},"max":{"1min":1.0740000000000001,"5min":1.47,"15min":1.47},"last":0.67200000000000004}]},{"osd":3,"last update":"Mon Jul 15 06:51:30 2024","interfaces":[{"interface":"back","average":{"1min":0.77400000000000002,"5min":0.79800000000000004,"15min":0.78000000000000003},"min":{"1min":0.372,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":1.0700000000000001,"5min":1.3220000000000001,"15min":1.3220000000000001},"last":0.77400000000000002},{"interface":"front","average":{"1min":0.71599999999999997,"5min":0.75600000000000001,"15min":0.72499999999999998},"min":{"1min":0.41999999999999998,"5min":0.377,"15min":0.377},"max":{"1min":1.044,"5min":1.431,"15min":1.431},"last":0.80700000000000005}]},{"osd":4,"last update":"Mon Jul 15 06:51:17 2024","interfaces":[{"interface":"back","average":{"1min":0.99299999999999999,"5min":1.03,"15min":0.95699999999999996},"min":{"1min":0.68799999999999994,"5min":0.66200000000000003,"15min":0.66200000000000003},"max":{"1min":1.226,"5min":1.5840000000000001,"15min":1.5840000000000001},"last":1.0409999999999999},{"interface":"front","average":{"1min":0.96599999999999997,"5min":0.96999999999999997,"15min":0.92800000000000005},"min":{"1min":0.747,"5min":0.60299999999999998,"15min":0.60299999999999998},"max":{"1min":1.21,"5min":1.6839999999999999,"15min":1.6839999999999999},"last":0.90700000000000003}]},{"osd":5,"last update":"Mon Jul 15 06:52:09 2024","interfaces":[{"interface":"back","average":{"1min":0.94799999999999995,"5min":0.96599999999999997,"15min":1.036},"min":{"1min":0.69399999999999995,"5min":0.63,"15min":0.63},"max":{"1min":1.375,"5min":1.5109999999999999,"15min":4.431},"last":0.96099999999999997},{"interface":"front","average":{"1min":0.94399999999999995,"5min":0.98299999999999998,"15min":1.022},"min":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"max":{"1min":1.173,"5min":1.4630000000000001,"15min":3.0670000000000002},"last":0.98699999999999999}]},{"osd":6,"last update":"Mon Jul 15 06:51:59 2024","interfaces":[{"interface":"back","average":{"1min":1.012,"5min":1.034,"15min":1.0620000000000001},"min":{"1min":0.624,"5min":0.624,"15min":0.624},"max":{"1min":1.5449999999999999,"5min":1.5620000000000001,"15min":2.0339999999999998},"last":1.0960000000000001},{"interface":"front","average":{"1min":1.0640000000000001,"5min":1.0389999999999999,"15min":1.081},"min":{"1min":0.66400000000000003,"5min":0.65200000000000002,"15min":0.65200000000000002},"max":{"1min":1.3500000000000001,"5min":1.617,"15min":1.897},"last":0.879}]},{"osd":7,"last update":"Mon Jul 15 06:51:57 2024","interfaces":[{"interface":"back","average":{"1min":1.0549999999999999,"5min":1.0960000000000001,"15min":1.1299999999999999},"min":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"max":{"1min":1.6140000000000001,"5min":1.988,"15min":1.988},"last":1.1359999999999999},{"interface":"front","average":{"1min":1.0629999999999999,"5min":1.0389999999999999,"15min":1.0609999999999999},"min":{"1min":0.64600000000000002,"5min":0.64200000000000002,"15min":0.64200000000000002},"max":{"1min":1.5329999999999999,"5min":1.7809999999999999,"15min":1.7809999999999999},"last":1.022}]}]},{"osd":0,"up_from":8,"seq":34359738510,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298644,"kb_used_data":1288,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446508,"statfs":{"total":95995035648,"available":95689224192,"internally_reserved":0,"allocated":1318912,"data_stored":904546,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Jul 15 06:51:42 2024","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":0.65200000000000002,"15min":0.58999999999999997},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.26900000000000002},"max":{"1min":0.875,"5min":1.0469999999999999,"15min":1.2070000000000001},"last":0.80800000000000005},{"interface":"front","average":{"1min":0.629,"5min":0.71999999999999997,"15min":0.64300000000000002},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":1.071,"5min":1.236,"15min":1.236},"last":0.64000000000000001}]},{"osd":2,"last update":"Mon Jul 15 06:51:32 2024","interfaces":[{"interface":"back","average":{"1min":0.72599999999999998,"5min":0.755,"15min":0.69099999999999995},"min":{"1min":0.435,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":1.1759999999999999,"5min":1.294,"15min":1.294},"last":0.83699999999999997},{"interface":"front","average":{"1min":0.71099999999999997,"5min":0.70899999999999996,"15min":0.67700000000000005},"min":{"1min":0.48299999999999998,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.98599999999999999,"5min":1.54,"15min":1.54},"last":0.78000000000000003}]},{"osd":3,"last update":"Mon Jul 15 06:51:24 2024","interfaces":[{"interface":"back","average":{"1min":0.84699999999999998,"5min":0.79900000000000004,"15min":0.77400000000000002},"min":{"1min":0.45900000000000002,"5min":0.371,"15min":0.371},"max":{"1min":1.252,"5min":1.7470000000000001,"15min":1.7470000000000001},"last":0.68799999999999994},{"interface":"front","average":{"1min":0.70399999999999996,"5min":0.73399999999999999,"15min":0.71699999999999997},"min":{"1min":0.40300000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":1.0489999999999999,"5min":1.0489999999999999,"15min":1.0489999999999999},"last":0.72699999999999998}]},{"osd":4,"last update":"Mon Jul 15 06:51:19 2024","interfaces":[{"interface":"back","average":{"1min":0.94199999999999995,"5min":0.95299999999999996,"15min":0.90700000000000003},"min":{"1min":0.746,"5min":0.51800000000000002,"15min":0.51800000000000002},"max":{"1min":1.2050000000000001,"5min":1.2909999999999999,"15min":1.2909999999999999},"last":1.089},{"interface":"front","average":{"1min":1.0369999999999999,"5min":1.018,"15min":0.94699999999999995},"min":{"1min":0.69199999999999995,"5min":0.52800000000000002,"15min":0.52800000000000002},"max":{"1min":1.518,"5min":1.518,"15min":1.518},"last":0.90700000000000003}]},{"osd":5,"last update":"Mon Jul 15 06:52:04 2024","interfaces":[{"interface":"back","average":{"1min":0.89500000000000002,"5min":0.91800000000000004,"15min":0.94099999999999995},"min":{"1min":0.59299999999999997,"5min":0.58399999999999996,"15min":0.58399999999999996},"max":{"1min":1.26,"5min":1.3049999999999999,"15min":1.3280000000000001},"last":1.0269999999999999},{"interface":"front","average":{"1min":0.93899999999999995,"5min":0.96199999999999997,"15min":0.95699999999999996},"min":{"1min":0.69099999999999995,"5min":0.51000000000000001,"15min":0.51000000000000001},"max":{"1min":1.4099999999999999,"5min":1.619,"15min":1.619},"last":0.98999999999999999}]},{"osd":6,"last update":"Mon Jul 15 06:52:04 2024","interfaces":[{"interface":"back","average":{"1min":0.95899999999999996,"5min":0.996,"15min":0.98099999999999998},"min":{"1min":0.57799999999999996,"5min":0.5,"15min":0.5},"max":{"1min":1.3740000000000001,"5min":1.3740000000000001,"15min":1.3740000000000001},"last":0.93200000000000005},{"interface":"front","average":{"1min":0.98299999999999998,"5min":1.0129999999999999,"15min":1.0369999999999999},"min":{"1min":0.621,"5min":0.47799999999999998,"15min":0.47799999999999998},"max":{"1min":1.5669999999999999,"5min":1.5669999999999999,"15min":1.5669999999999999},"last":1.1240000000000001}]},{"osd":7,"last update":"Mon Jul 15 06:51:53 2024","interfaces":[{"interface":"back","average":{"1min":0.996,"5min":1.048,"15min":1.048},"min":{"1min":0.67400000000000004,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":1.4650000000000001,"5min":1.4650000000000001,"15min":1.4650000000000001},"last":1.1759999999999999},{"interface":"front","average":{"1min":0.998,"5min":1.0600000000000001,"15min":1.052},"min":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"max":{"1min":1.518,"5min":1.827,"15min":1.827},"last":1.1899999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1698,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1622,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-07-15T06:52:11.125 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph pg dump --format=json 2024-07-15T06:52:12.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:52:11 smithi134 bash[24068]: cluster 2024-07-15T06:52:10.188475+0000 mgr.y (mgr.24272) 172 : cluster [DBG] pgmap v147: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:52:12.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:52:11 smithi134 bash[24068]: audit 2024-07-15T06:52:10.383101+0000 mgr.y (mgr.24272) 173 : audit [DBG] from='client.14709 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-07-15T06:52:12.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:52:11 smithi089 bash[21793]: cluster 2024-07-15T06:52:10.188475+0000 mgr.y (mgr.24272) 172 : cluster [DBG] pgmap v147: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:52:12.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:52:11 smithi089 bash[21793]: audit 2024-07-15T06:52:10.383101+0000 mgr.y (mgr.24272) 173 : audit [DBG] from='client.14709 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-07-15T06:52:12.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:52:11 smithi089 bash[31372]: cluster 2024-07-15T06:52:10.188475+0000 mgr.y (mgr.24272) 172 : cluster [DBG] pgmap v147: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:52:12.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:52:11 smithi089 bash[31372]: audit 2024-07-15T06:52:10.383101+0000 mgr.y (mgr.24272) 173 : audit [DBG] from='client.14709 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-07-15T06:52:13.119 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:52:12 smithi089 bash[22045]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:52:12] "GET /metrics HTTP/1.1" 200 239255 "" "Prometheus/2.43.0" 2024-07-15T06:52:14.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:52:13 smithi134 bash[24068]: cluster 2024-07-15T06:52:12.189056+0000 mgr.y (mgr.24272) 174 : cluster [DBG] pgmap v148: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:52:14.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:52:13 smithi089 bash[21793]: cluster 2024-07-15T06:52:12.189056+0000 mgr.y (mgr.24272) 174 : cluster [DBG] pgmap v148: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:52:14.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:52:13 smithi089 bash[31372]: cluster 2024-07-15T06:52:12.189056+0000 mgr.y (mgr.24272) 174 : cluster [DBG] pgmap v148: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:52:15.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:52:14 smithi134 bash[24068]: audit 2024-07-15T06:52:14.194251+0000 mon.a (mon.0) 896 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:52:15.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:52:14 smithi089 bash[21793]: audit 2024-07-15T06:52:14.194251+0000 mon.a (mon.0) 896 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:52:15.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:52:14 smithi089 bash[31372]: audit 2024-07-15T06:52:14.194251+0000 mon.a (mon.0) 896 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:52:15.909 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T06:52:16.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:52:15 smithi089 bash[21793]: cluster 2024-07-15T06:52:14.189754+0000 mgr.y (mgr.24272) 175 : cluster [DBG] pgmap v149: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:52:16.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:52:15 smithi089 bash[31372]: cluster 2024-07-15T06:52:14.189754+0000 mgr.y (mgr.24272) 175 : cluster [DBG] pgmap v149: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:52:16.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:52:15 smithi134 bash[24068]: cluster 2024-07-15T06:52:14.189754+0000 mgr.y (mgr.24272) 175 : cluster [DBG] pgmap v149: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:52:17.117 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:52:16 smithi089 bash[31372]: cluster 2024-07-15T06:52:16.190838+0000 mgr.y (mgr.24272) 176 : cluster [DBG] pgmap v150: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:52:17.117 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:52:16 smithi089 bash[21793]: cluster 2024-07-15T06:52:16.190838+0000 mgr.y (mgr.24272) 176 : cluster [DBG] pgmap v150: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:52:17.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:52:16 smithi134 bash[24068]: cluster 2024-07-15T06:52:16.190838+0000 mgr.y (mgr.24272) 176 : cluster [DBG] pgmap v150: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:52:17.305 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:52:16 smithi134 bash[25019]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:52:16] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-07-15T06:52:19.369 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:52:19 smithi089 bash[31372]: cluster 2024-07-15T06:52:18.191487+0000 mgr.y (mgr.24272) 177 : cluster [DBG] pgmap v151: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:52:19.369 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:52:19 smithi089 bash[21793]: cluster 2024-07-15T06:52:18.191487+0000 mgr.y (mgr.24272) 177 : cluster [DBG] pgmap v151: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:52:19.386 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-07-15T06:52:19.387 INFO:teuthology.orchestra.run.smithi089.stderr:dumped all 2024-07-15T06:52:19.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:52:19 smithi134 bash[24068]: cluster 2024-07-15T06:52:18.191487+0000 mgr.y (mgr.24272) 177 : cluster [DBG] pgmap v151: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:52:20.148 INFO:teuthology.orchestra.run.smithi089.stdout:{"pg_ready":true,"pg_map":{"version":151,"stamp":"2024-07-15T06:52:18.191142+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592706,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":259,"num_read_kb":355,"num_write":224,"num_write_kb":2429,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":173,"ondisk_log_size":173,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2386292,"kb_used_data":7428,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747574924,"statfs":{"total":767960285184,"available":765516722176,"internally_reserved":0,"allocated":7606272,"data_stored":4286160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004604"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.803509+0000","last_change":"2024-07-15T06:50:29.852535+0000","last_active":"2024-07-15T06:50:44.803509+0000","last_peered":"2024-07-15T06:50:44.803509+0000","last_clean":"2024-07-15T06:50:44.803509+0000","last_became_active":"2024-07-15T06:50:29.851305+0000","last_became_peered":"2024-07-15T06:50:29.851305+0000","last_unstale":"2024-07-15T06:50:44.803509+0000","last_undegraded":"2024-07-15T06:50:44.803509+0000","last_fullsized":"2024-07-15T06:50:44.803509+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T11:34:38.540931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":18,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.804032+0000","last_change":"2024-07-15T06:50:29.262370+0000","last_active":"2024-07-15T06:50:44.804032+0000","last_peered":"2024-07-15T06:50:44.804032+0000","last_clean":"2024-07-15T06:50:44.804032+0000","last_became_active":"2024-07-15T06:50:29.261755+0000","last_became_peered":"2024-07-15T06:50:29.261755+0000","last_unstale":"2024-07-15T06:50:44.804032+0000","last_undegraded":"2024-07-15T06:50:44.804032+0000","last_fullsized":"2024-07-15T06:50:44.804032+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T17:22:35.393499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":7,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.790474+0000","last_change":"2024-07-15T06:50:29.851103+0000","last_active":"2024-07-15T06:50:44.790474+0000","last_peered":"2024-07-15T06:50:44.790474+0000","last_clean":"2024-07-15T06:50:44.790474+0000","last_became_active":"2024-07-15T06:50:29.850222+0000","last_became_peered":"2024-07-15T06:50:29.850222+0000","last_unstale":"2024-07-15T06:50:44.790474+0000","last_undegraded":"2024-07-15T06:50:44.790474+0000","last_fullsized":"2024-07-15T06:50:44.790474+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T09:44:30.500327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.790700+0000","last_change":"2024-07-15T06:50:29.275925+0000","last_active":"2024-07-15T06:50:44.790700+0000","last_peered":"2024-07-15T06:50:44.790700+0000","last_clean":"2024-07-15T06:50:44.790700+0000","last_became_active":"2024-07-15T06:50:29.275169+0000","last_became_peered":"2024-07-15T06:50:29.275169+0000","last_unstale":"2024-07-15T06:50:44.790700+0000","last_undegraded":"2024-07-15T06:50:44.790700+0000","last_fullsized":"2024-07-15T06:50:44.790700+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T17:59:53.279149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.790532+0000","last_change":"2024-07-15T06:50:29.852471+0000","last_active":"2024-07-15T06:50:44.790532+0000","last_peered":"2024-07-15T06:50:44.790532+0000","last_clean":"2024-07-15T06:50:44.790532+0000","last_became_active":"2024-07-15T06:50:29.851082+0000","last_became_peered":"2024-07-15T06:50:29.851082+0000","last_unstale":"2024-07-15T06:50:44.790532+0000","last_undegraded":"2024-07-15T06:50:44.790532+0000","last_fullsized":"2024-07-15T06:50:44.790532+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T14:46:50.862472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.803607+0000","last_change":"2024-07-15T06:50:29.262044+0000","last_active":"2024-07-15T06:50:44.803607+0000","last_peered":"2024-07-15T06:50:44.803607+0000","last_clean":"2024-07-15T06:50:44.803607+0000","last_became_active":"2024-07-15T06:50:29.261056+0000","last_became_peered":"2024-07-15T06:50:29.261056+0000","last_unstale":"2024-07-15T06:50:44.803607+0000","last_undegraded":"2024-07-15T06:50:44.803607+0000","last_fullsized":"2024-07-15T06:50:44.803607+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T08:13:43.705862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.803686+0000","last_change":"2024-07-15T06:50:29.849048+0000","last_active":"2024-07-15T06:50:44.803686+0000","last_peered":"2024-07-15T06:50:44.803686+0000","last_clean":"2024-07-15T06:50:44.803686+0000","last_became_active":"2024-07-15T06:50:29.848064+0000","last_became_peered":"2024-07-15T06:50:29.848064+0000","last_unstale":"2024-07-15T06:50:44.803686+0000","last_undegraded":"2024-07-15T06:50:44.803686+0000","last_fullsized":"2024-07-15T06:50:44.803686+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T17:10:05.793024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":26,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.797343+0000","last_change":"2024-07-15T06:50:29.251566+0000","last_active":"2024-07-15T06:50:44.797343+0000","last_peered":"2024-07-15T06:50:44.797343+0000","last_clean":"2024-07-15T06:50:44.797343+0000","last_became_active":"2024-07-15T06:50:29.250951+0000","last_became_peered":"2024-07-15T06:50:29.250951+0000","last_unstale":"2024-07-15T06:50:44.797343+0000","last_undegraded":"2024-07-15T06:50:44.797343+0000","last_fullsized":"2024-07-15T06:50:44.797343+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T15:44:37.697435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":12,"num_read_kb":12,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":16,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.804756+0000","last_change":"2024-07-15T06:50:29.269211+0000","last_active":"2024-07-15T06:50:44.804756+0000","last_peered":"2024-07-15T06:50:44.804756+0000","last_clean":"2024-07-15T06:50:44.804756+0000","last_became_active":"2024-07-15T06:50:29.268466+0000","last_became_peered":"2024-07-15T06:50:29.268466+0000","last_unstale":"2024-07-15T06:50:44.804756+0000","last_undegraded":"2024-07-15T06:50:44.804756+0000","last_fullsized":"2024-07-15T06:50:44.804756+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T10:28:35.703616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.790094+0000","last_change":"2024-07-15T06:50:29.845162+0000","last_active":"2024-07-15T06:50:44.790094+0000","last_peered":"2024-07-15T06:50:44.790094+0000","last_clean":"2024-07-15T06:50:44.790094+0000","last_became_active":"2024-07-15T06:50:29.844204+0000","last_became_peered":"2024-07-15T06:50:29.844204+0000","last_unstale":"2024-07-15T06:50:44.790094+0000","last_undegraded":"2024-07-15T06:50:44.790094+0000","last_fullsized":"2024-07-15T06:50:44.790094+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T16:56:12.605992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":17,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.804896+0000","last_change":"2024-07-15T06:50:29.273685+0000","last_active":"2024-07-15T06:50:44.804896+0000","last_peered":"2024-07-15T06:50:44.804896+0000","last_clean":"2024-07-15T06:50:44.804896+0000","last_became_active":"2024-07-15T06:50:29.272972+0000","last_became_peered":"2024-07-15T06:50:29.272972+0000","last_unstale":"2024-07-15T06:50:44.804896+0000","last_undegraded":"2024-07-15T06:50:44.804896+0000","last_fullsized":"2024-07-15T06:50:44.804896+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T08:42:09.115844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.797509+0000","last_change":"2024-07-15T06:50:29.303167+0000","last_active":"2024-07-15T06:50:44.797509+0000","last_peered":"2024-07-15T06:50:44.797509+0000","last_clean":"2024-07-15T06:50:44.797509+0000","last_became_active":"2024-07-15T06:50:29.302500+0000","last_became_peered":"2024-07-15T06:50:29.302500+0000","last_unstale":"2024-07-15T06:50:44.797509+0000","last_undegraded":"2024-07-15T06:50:44.797509+0000","last_fullsized":"2024-07-15T06:50:44.797509+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T12:32:35.025536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":16,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.804760+0000","last_change":"2024-07-15T06:50:29.271443+0000","last_active":"2024-07-15T06:50:44.804760+0000","last_peered":"2024-07-15T06:50:44.804760+0000","last_clean":"2024-07-15T06:50:44.804760+0000","last_became_active":"2024-07-15T06:50:29.270851+0000","last_became_peered":"2024-07-15T06:50:29.270851+0000","last_unstale":"2024-07-15T06:50:44.804760+0000","last_undegraded":"2024-07-15T06:50:44.804760+0000","last_fullsized":"2024-07-15T06:50:44.804760+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T14:45:12.733929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":23,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-07-15T06:50:45.797074+0000","last_change":"2024-07-15T06:50:45.797074+0000","last_active":"2024-07-15T06:50:45.797074+0000","last_peered":"2024-07-15T06:50:45.797074+0000","last_clean":"2024-07-15T06:50:45.797074+0000","last_became_active":"2024-07-15T06:50:45.796066+0000","last_became_peered":"2024-07-15T06:50:45.796066+0000","last_unstale":"2024-07-15T06:50:45.797074+0000","last_undegraded":"2024-07-15T06:50:45.797074+0000","last_fullsized":"2024-07-15T06:50:45.797074+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T09:53:48.422246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.790646+0000","last_change":"2024-07-15T06:50:29.274245+0000","last_active":"2024-07-15T06:50:44.790646+0000","last_peered":"2024-07-15T06:50:44.790646+0000","last_clean":"2024-07-15T06:50:44.790646+0000","last_became_active":"2024-07-15T06:50:29.273642+0000","last_became_peered":"2024-07-15T06:50:29.273642+0000","last_unstale":"2024-07-15T06:50:44.790646+0000","last_undegraded":"2024-07-15T06:50:44.790646+0000","last_fullsized":"2024-07-15T06:50:44.790646+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T09:06:19.595465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.790771+0000","last_change":"2024-07-15T06:50:29.304844+0000","last_active":"2024-07-15T06:50:44.790771+0000","last_peered":"2024-07-15T06:50:44.790771+0000","last_clean":"2024-07-15T06:50:44.790771+0000","last_became_active":"2024-07-15T06:50:29.304368+0000","last_became_peered":"2024-07-15T06:50:29.304368+0000","last_unstale":"2024-07-15T06:50:44.790771+0000","last_undegraded":"2024-07-15T06:50:44.790771+0000","last_fullsized":"2024-07-15T06:50:44.790771+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T07:52:24.418782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"68'3","reported_seq":18,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.802899+0000","last_change":"2024-07-15T06:50:29.269551+0000","last_active":"2024-07-15T06:50:44.802899+0000","last_peered":"2024-07-15T06:50:44.802899+0000","last_clean":"2024-07-15T06:50:44.802899+0000","last_became_active":"2024-07-15T06:50:29.268783+0000","last_became_peered":"2024-07-15T06:50:29.268783+0000","last_unstale":"2024-07-15T06:50:44.802899+0000","last_undegraded":"2024-07-15T06:50:44.802899+0000","last_fullsized":"2024-07-15T06:50:44.802899+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T06:57:29.304742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.798055+0000","last_change":"2024-07-15T06:50:29.299475+0000","last_active":"2024-07-15T06:50:44.798055+0000","last_peered":"2024-07-15T06:50:44.798055+0000","last_clean":"2024-07-15T06:50:44.798055+0000","last_became_active":"2024-07-15T06:50:29.298804+0000","last_became_peered":"2024-07-15T06:50:29.298804+0000","last_unstale":"2024-07-15T06:50:44.798055+0000","last_undegraded":"2024-07-15T06:50:44.798055+0000","last_fullsized":"2024-07-15T06:50:44.798055+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T11:01:29.878472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.800793+0000","last_change":"2024-07-15T06:50:29.254893+0000","last_active":"2024-07-15T06:50:44.800793+0000","last_peered":"2024-07-15T06:50:44.800793+0000","last_clean":"2024-07-15T06:50:44.800793+0000","last_became_active":"2024-07-15T06:50:29.254025+0000","last_became_peered":"2024-07-15T06:50:29.254025+0000","last_unstale":"2024-07-15T06:50:44.800793+0000","last_undegraded":"2024-07-15T06:50:44.800793+0000","last_fullsized":"2024-07-15T06:50:44.800793+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T16:38:14.132555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.798543+0000","last_change":"2024-07-15T06:50:29.301409+0000","last_active":"2024-07-15T06:50:44.798543+0000","last_peered":"2024-07-15T06:50:44.798543+0000","last_clean":"2024-07-15T06:50:44.798543+0000","last_became_active":"2024-07-15T06:50:29.300988+0000","last_became_peered":"2024-07-15T06:50:29.300988+0000","last_unstale":"2024-07-15T06:50:44.798543+0000","last_undegraded":"2024-07-15T06:50:44.798543+0000","last_fullsized":"2024-07-15T06:50:44.798543+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T14:40:46.878950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.797901+0000","last_change":"2024-07-15T06:50:29.257021+0000","last_active":"2024-07-15T06:50:44.797901+0000","last_peered":"2024-07-15T06:50:44.797901+0000","last_clean":"2024-07-15T06:50:44.797901+0000","last_became_active":"2024-07-15T06:50:29.256202+0000","last_became_peered":"2024-07-15T06:50:29.256202+0000","last_unstale":"2024-07-15T06:50:44.797901+0000","last_undegraded":"2024-07-15T06:50:44.797901+0000","last_fullsized":"2024-07-15T06:50:44.797901+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T18:27:17.492902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.798748+0000","last_change":"2024-07-15T06:50:29.302450+0000","last_active":"2024-07-15T06:50:44.798748+0000","last_peered":"2024-07-15T06:50:44.798748+0000","last_clean":"2024-07-15T06:50:44.798748+0000","last_became_active":"2024-07-15T06:50:29.302017+0000","last_became_peered":"2024-07-15T06:50:29.302017+0000","last_unstale":"2024-07-15T06:50:44.798748+0000","last_undegraded":"2024-07-15T06:50:44.798748+0000","last_fullsized":"2024-07-15T06:50:44.798748+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T15:31:08.157676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.802186+0000","last_change":"2024-07-15T06:50:29.277528+0000","last_active":"2024-07-15T06:50:44.802186+0000","last_peered":"2024-07-15T06:50:44.802186+0000","last_clean":"2024-07-15T06:50:44.802186+0000","last_became_active":"2024-07-15T06:50:29.277057+0000","last_became_peered":"2024-07-15T06:50:29.277057+0000","last_unstale":"2024-07-15T06:50:44.802186+0000","last_undegraded":"2024-07-15T06:50:44.802186+0000","last_fullsized":"2024-07-15T06:50:44.802186+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T09:04:06.474542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.795540+0000","last_change":"2024-07-15T06:50:29.311569+0000","last_active":"2024-07-15T06:50:44.795540+0000","last_peered":"2024-07-15T06:50:44.795540+0000","last_clean":"2024-07-15T06:50:44.795540+0000","last_became_active":"2024-07-15T06:50:29.311002+0000","last_became_peered":"2024-07-15T06:50:29.311002+0000","last_unstale":"2024-07-15T06:50:44.795540+0000","last_undegraded":"2024-07-15T06:50:44.795540+0000","last_fullsized":"2024-07-15T06:50:44.795540+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T17:51:08.575282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"59'128","reported_seq":277,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.803597+0000","last_change":"2024-07-15T06:46:49.356522+0000","last_active":"2024-07-15T06:50:44.803597+0000","last_peered":"2024-07-15T06:50:44.803597+0000","last_clean":"2024-07-15T06:50:44.803597+0000","last_became_active":"2024-07-15T06:46:49.355501+0000","last_became_peered":"2024-07-15T06:46:49.355501+0000","last_unstale":"2024-07-15T06:50:44.803597+0000","last_undegraded":"2024-07-15T06:50:44.803597+0000","last_fullsized":"2024-07-15T06:50:44.803597+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:42:21.582843+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:42:21.582843+0000","last_clean_scrub_stamp":"2024-07-15T06:42:21.582843+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T10:17:36.460676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":18,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.790563+0000","last_change":"2024-07-15T06:48:28.672045+0000","last_active":"2024-07-15T06:50:44.790563+0000","last_peered":"2024-07-15T06:50:44.790563+0000","last_clean":"2024-07-15T06:50:44.790563+0000","last_became_active":"2024-07-15T06:48:28.671290+0000","last_became_peered":"2024-07-15T06:48:28.671290+0000","last_unstale":"2024-07-15T06:50:44.790563+0000","last_undegraded":"2024-07-15T06:50:44.790563+0000","last_fullsized":"2024-07-15T06:50:44.790563+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_clean_scrub_stamp":"2024-07-15T06:48:27.655327+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T11:10:59.552676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.802607+0000","last_change":"2024-07-15T06:50:29.277850+0000","last_active":"2024-07-15T06:50:44.802607+0000","last_peered":"2024-07-15T06:50:44.802607+0000","last_clean":"2024-07-15T06:50:44.802607+0000","last_became_active":"2024-07-15T06:50:29.276384+0000","last_became_peered":"2024-07-15T06:50:29.276384+0000","last_unstale":"2024-07-15T06:50:44.802607+0000","last_undegraded":"2024-07-15T06:50:44.802607+0000","last_fullsized":"2024-07-15T06:50:44.802607+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T15:57:11.793888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.801412+0000","last_change":"2024-07-15T06:50:29.851397+0000","last_active":"2024-07-15T06:50:44.801412+0000","last_peered":"2024-07-15T06:50:44.801412+0000","last_clean":"2024-07-15T06:50:44.801412+0000","last_became_active":"2024-07-15T06:50:29.850009+0000","last_became_peered":"2024-07-15T06:50:29.850009+0000","last_unstale":"2024-07-15T06:50:44.801412+0000","last_undegraded":"2024-07-15T06:50:44.801412+0000","last_fullsized":"2024-07-15T06:50:44.801412+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T08:04:19.006397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":18,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.803453+0000","last_change":"2024-07-15T06:48:28.737091+0000","last_active":"2024-07-15T06:50:44.803453+0000","last_peered":"2024-07-15T06:50:44.803453+0000","last_clean":"2024-07-15T06:50:44.803453+0000","last_became_active":"2024-07-15T06:48:28.736574+0000","last_became_peered":"2024-07-15T06:48:28.736574+0000","last_unstale":"2024-07-15T06:50:44.803453+0000","last_undegraded":"2024-07-15T06:50:44.803453+0000","last_fullsized":"2024-07-15T06:50:44.803453+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_clean_scrub_stamp":"2024-07-15T06:48:27.655327+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T09:31:07.195677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.789928+0000","last_change":"2024-07-15T06:50:29.261545+0000","last_active":"2024-07-15T06:50:44.789928+0000","last_peered":"2024-07-15T06:50:44.789928+0000","last_clean":"2024-07-15T06:50:44.789928+0000","last_became_active":"2024-07-15T06:50:29.261213+0000","last_became_peered":"2024-07-15T06:50:29.261213+0000","last_unstale":"2024-07-15T06:50:44.789928+0000","last_undegraded":"2024-07-15T06:50:44.789928+0000","last_fullsized":"2024-07-15T06:50:44.789928+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T10:34:24.671609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.790362+0000","last_change":"2024-07-15T06:50:29.303396+0000","last_active":"2024-07-15T06:50:44.790362+0000","last_peered":"2024-07-15T06:50:44.790362+0000","last_clean":"2024-07-15T06:50:44.790362+0000","last_became_active":"2024-07-15T06:50:29.302950+0000","last_became_peered":"2024-07-15T06:50:29.302950+0000","last_unstale":"2024-07-15T06:50:44.790362+0000","last_undegraded":"2024-07-15T06:50:44.790362+0000","last_fullsized":"2024-07-15T06:50:44.790362+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T09:41:19.686254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":18,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.798119+0000","last_change":"2024-07-15T06:48:28.738441+0000","last_active":"2024-07-15T06:50:44.798119+0000","last_peered":"2024-07-15T06:50:44.798119+0000","last_clean":"2024-07-15T06:50:44.798119+0000","last_became_active":"2024-07-15T06:48:28.737053+0000","last_became_peered":"2024-07-15T06:48:28.737053+0000","last_unstale":"2024-07-15T06:50:44.798119+0000","last_undegraded":"2024-07-15T06:50:44.798119+0000","last_fullsized":"2024-07-15T06:50:44.798119+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_clean_scrub_stamp":"2024-07-15T06:48:27.655327+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T08:33:33.092342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.792499+0000","last_change":"2024-07-15T06:50:29.249905+0000","last_active":"2024-07-15T06:50:44.792499+0000","last_peered":"2024-07-15T06:50:44.792499+0000","last_clean":"2024-07-15T06:50:44.792499+0000","last_became_active":"2024-07-15T06:50:29.249163+0000","last_became_peered":"2024-07-15T06:50:29.249163+0000","last_unstale":"2024-07-15T06:50:44.792499+0000","last_undegraded":"2024-07-15T06:50:44.792499+0000","last_fullsized":"2024-07-15T06:50:44.792499+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T11:48:19.218891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.795525+0000","last_change":"2024-07-15T06:50:29.304233+0000","last_active":"2024-07-15T06:50:44.795525+0000","last_peered":"2024-07-15T06:50:44.795525+0000","last_clean":"2024-07-15T06:50:44.795525+0000","last_became_active":"2024-07-15T06:50:29.303966+0000","last_became_peered":"2024-07-15T06:50:29.303966+0000","last_unstale":"2024-07-15T06:50:44.795525+0000","last_undegraded":"2024-07-15T06:50:44.795525+0000","last_fullsized":"2024-07-15T06:50:44.795525+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T17:22:58.514985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":18,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.790222+0000","last_change":"2024-07-15T06:48:28.674943+0000","last_active":"2024-07-15T06:50:44.790222+0000","last_peered":"2024-07-15T06:50:44.790222+0000","last_clean":"2024-07-15T06:50:44.790222+0000","last_became_active":"2024-07-15T06:48:28.674174+0000","last_became_peered":"2024-07-15T06:48:28.674174+0000","last_unstale":"2024-07-15T06:50:44.790222+0000","last_undegraded":"2024-07-15T06:50:44.790222+0000","last_fullsized":"2024-07-15T06:50:44.790222+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_clean_scrub_stamp":"2024-07-15T06:48:27.655327+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T17:40:56.394801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.800666+0000","last_change":"2024-07-15T06:50:29.258941+0000","last_active":"2024-07-15T06:50:44.800666+0000","last_peered":"2024-07-15T06:50:44.800666+0000","last_clean":"2024-07-15T06:50:44.800666+0000","last_became_active":"2024-07-15T06:50:29.258207+0000","last_became_peered":"2024-07-15T06:50:29.258207+0000","last_unstale":"2024-07-15T06:50:44.800666+0000","last_undegraded":"2024-07-15T06:50:44.800666+0000","last_fullsized":"2024-07-15T06:50:44.800666+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T14:30:34.638511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.794413+0000","last_change":"2024-07-15T06:50:29.311630+0000","last_active":"2024-07-15T06:50:44.794413+0000","last_peered":"2024-07-15T06:50:44.794413+0000","last_clean":"2024-07-15T06:50:44.794413+0000","last_became_active":"2024-07-15T06:50:29.311134+0000","last_became_peered":"2024-07-15T06:50:29.311134+0000","last_unstale":"2024-07-15T06:50:44.794413+0000","last_undegraded":"2024-07-15T06:50:44.794413+0000","last_fullsized":"2024-07-15T06:50:44.794413+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T16:20:39.971632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.790692+0000","last_change":"2024-07-15T06:50:29.268874+0000","last_active":"2024-07-15T06:50:44.790692+0000","last_peered":"2024-07-15T06:50:44.790692+0000","last_clean":"2024-07-15T06:50:44.790692+0000","last_became_active":"2024-07-15T06:50:29.268302+0000","last_became_peered":"2024-07-15T06:50:29.268302+0000","last_unstale":"2024-07-15T06:50:44.790692+0000","last_undegraded":"2024-07-15T06:50:44.790692+0000","last_fullsized":"2024-07-15T06:50:44.790692+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T16:39:58.460334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":18,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.795107+0000","last_change":"2024-07-15T06:48:28.738368+0000","last_active":"2024-07-15T06:50:44.795107+0000","last_peered":"2024-07-15T06:50:44.795107+0000","last_clean":"2024-07-15T06:50:44.795107+0000","last_became_active":"2024-07-15T06:48:28.737382+0000","last_became_peered":"2024-07-15T06:48:28.737382+0000","last_unstale":"2024-07-15T06:50:44.795107+0000","last_undegraded":"2024-07-15T06:50:44.795107+0000","last_fullsized":"2024-07-15T06:50:44.795107+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_clean_scrub_stamp":"2024-07-15T06:48:27.655327+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T17:01:42.558382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.793570+0000","last_change":"2024-07-15T06:50:29.310565+0000","last_active":"2024-07-15T06:50:44.793570+0000","last_peered":"2024-07-15T06:50:44.793570+0000","last_clean":"2024-07-15T06:50:44.793570+0000","last_became_active":"2024-07-15T06:50:29.310142+0000","last_became_peered":"2024-07-15T06:50:29.310142+0000","last_unstale":"2024-07-15T06:50:44.793570+0000","last_undegraded":"2024-07-15T06:50:44.793570+0000","last_fullsized":"2024-07-15T06:50:44.793570+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T17:53:38.506736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.795518+0000","last_change":"2024-07-15T06:50:29.273932+0000","last_active":"2024-07-15T06:50:44.795518+0000","last_peered":"2024-07-15T06:50:44.795518+0000","last_clean":"2024-07-15T06:50:44.795518+0000","last_became_active":"2024-07-15T06:50:29.273266+0000","last_became_peered":"2024-07-15T06:50:29.273266+0000","last_unstale":"2024-07-15T06:50:44.795518+0000","last_undegraded":"2024-07-15T06:50:44.795518+0000","last_fullsized":"2024-07-15T06:50:44.795518+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T07:02:37.137230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":18,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.804934+0000","last_change":"2024-07-15T06:48:28.872322+0000","last_active":"2024-07-15T06:50:44.804934+0000","last_peered":"2024-07-15T06:50:44.804934+0000","last_clean":"2024-07-15T06:50:44.804934+0000","last_became_active":"2024-07-15T06:48:28.871320+0000","last_became_peered":"2024-07-15T06:48:28.871320+0000","last_unstale":"2024-07-15T06:50:44.804934+0000","last_undegraded":"2024-07-15T06:50:44.804934+0000","last_fullsized":"2024-07-15T06:50:44.804934+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_clean_scrub_stamp":"2024-07-15T06:48:27.655327+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T14:24:12.762681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.789936+0000","last_change":"2024-07-15T06:50:29.846557+0000","last_active":"2024-07-15T06:50:44.789936+0000","last_peered":"2024-07-15T06:50:44.789936+0000","last_clean":"2024-07-15T06:50:44.789936+0000","last_became_active":"2024-07-15T06:50:29.845667+0000","last_became_peered":"2024-07-15T06:50:29.845667+0000","last_unstale":"2024-07-15T06:50:44.789936+0000","last_undegraded":"2024-07-15T06:50:44.789936+0000","last_fullsized":"2024-07-15T06:50:44.789936+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T15:17:50.597165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":17,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.803808+0000","last_change":"2024-07-15T06:50:29.278450+0000","last_active":"2024-07-15T06:50:44.803808+0000","last_peered":"2024-07-15T06:50:44.803808+0000","last_clean":"2024-07-15T06:50:44.803808+0000","last_became_active":"2024-07-15T06:50:29.277938+0000","last_became_peered":"2024-07-15T06:50:29.277938+0000","last_unstale":"2024-07-15T06:50:44.803808+0000","last_undegraded":"2024-07-15T06:50:44.803808+0000","last_fullsized":"2024-07-15T06:50:44.803808+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T15:20:08.735416+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":18,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.794976+0000","last_change":"2024-07-15T06:48:28.872731+0000","last_active":"2024-07-15T06:50:44.794976+0000","last_peered":"2024-07-15T06:50:44.794976+0000","last_clean":"2024-07-15T06:50:44.794976+0000","last_became_active":"2024-07-15T06:48:28.871744+0000","last_became_peered":"2024-07-15T06:48:28.871744+0000","last_unstale":"2024-07-15T06:50:44.794976+0000","last_undegraded":"2024-07-15T06:50:44.794976+0000","last_fullsized":"2024-07-15T06:50:44.794976+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_clean_scrub_stamp":"2024-07-15T06:48:27.655327+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T11:36:09.487214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.801911+0000","last_change":"2024-07-15T06:50:29.848965+0000","last_active":"2024-07-15T06:50:44.801911+0000","last_peered":"2024-07-15T06:50:44.801911+0000","last_clean":"2024-07-15T06:50:44.801911+0000","last_became_active":"2024-07-15T06:50:29.848047+0000","last_became_peered":"2024-07-15T06:50:29.848047+0000","last_unstale":"2024-07-15T06:50:44.801911+0000","last_undegraded":"2024-07-15T06:50:44.801911+0000","last_fullsized":"2024-07-15T06:50:44.801911+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T15:48:12.470869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.790668+0000","last_change":"2024-07-15T06:50:29.263092+0000","last_active":"2024-07-15T06:50:44.790668+0000","last_peered":"2024-07-15T06:50:44.790668+0000","last_clean":"2024-07-15T06:50:44.790668+0000","last_became_active":"2024-07-15T06:50:29.262201+0000","last_became_peered":"2024-07-15T06:50:29.262201+0000","last_unstale":"2024-07-15T06:50:44.790668+0000","last_undegraded":"2024-07-15T06:50:44.790668+0000","last_fullsized":"2024-07-15T06:50:44.790668+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T13:14:27.912283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":18,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.802817+0000","last_change":"2024-07-15T06:48:28.675814+0000","last_active":"2024-07-15T06:50:44.802817+0000","last_peered":"2024-07-15T06:50:44.802817+0000","last_clean":"2024-07-15T06:50:44.802817+0000","last_became_active":"2024-07-15T06:48:28.674798+0000","last_became_peered":"2024-07-15T06:48:28.674798+0000","last_unstale":"2024-07-15T06:50:44.802817+0000","last_undegraded":"2024-07-15T06:50:44.802817+0000","last_fullsized":"2024-07-15T06:50:44.802817+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_clean_scrub_stamp":"2024-07-15T06:48:27.655327+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T14:36:19.382840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.797720+0000","last_change":"2024-07-15T06:50:29.298133+0000","last_active":"2024-07-15T06:50:44.797720+0000","last_peered":"2024-07-15T06:50:44.797720+0000","last_clean":"2024-07-15T06:50:44.797720+0000","last_became_active":"2024-07-15T06:50:29.297265+0000","last_became_peered":"2024-07-15T06:50:29.297265+0000","last_unstale":"2024-07-15T06:50:44.797720+0000","last_undegraded":"2024-07-15T06:50:44.797720+0000","last_fullsized":"2024-07-15T06:50:44.797720+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T08:21:53.743122+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":23,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-07-15T06:50:45.803975+0000","last_change":"2024-07-15T06:50:45.803975+0000","last_active":"2024-07-15T06:50:45.803975+0000","last_peered":"2024-07-15T06:50:45.803975+0000","last_clean":"2024-07-15T06:50:45.803975+0000","last_became_active":"2024-07-15T06:50:45.802915+0000","last_became_peered":"2024-07-15T06:50:45.802915+0000","last_unstale":"2024-07-15T06:50:45.803975+0000","last_undegraded":"2024-07-15T06:50:45.803975+0000","last_fullsized":"2024-07-15T06:50:45.803975+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T15:01:37.417772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"68'7","reported_seq":26,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.798196+0000","last_change":"2024-07-15T06:50:29.256120+0000","last_active":"2024-07-15T06:50:44.798196+0000","last_peered":"2024-07-15T06:50:44.798196+0000","last_clean":"2024-07-15T06:50:44.798196+0000","last_became_active":"2024-07-15T06:50:29.252769+0000","last_became_peered":"2024-07-15T06:50:29.252769+0000","last_unstale":"2024-07-15T06:50:44.798196+0000","last_undegraded":"2024-07-15T06:50:44.798196+0000","last_fullsized":"2024-07-15T06:50:44.798196+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T18:30:32.236566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1552,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":7,"num_read_kb":6,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.793928+0000","last_change":"2024-07-15T06:50:29.311811+0000","last_active":"2024-07-15T06:50:44.793928+0000","last_peered":"2024-07-15T06:50:44.793928+0000","last_clean":"2024-07-15T06:50:44.793928+0000","last_became_active":"2024-07-15T06:50:29.311460+0000","last_became_peered":"2024-07-15T06:50:29.311460+0000","last_unstale":"2024-07-15T06:50:44.793928+0000","last_undegraded":"2024-07-15T06:50:44.793928+0000","last_fullsized":"2024-07-15T06:50:44.793928+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T18:35:18.320094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":19,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.804389+0000","last_change":"2024-07-15T06:50:29.273640+0000","last_active":"2024-07-15T06:50:44.804389+0000","last_peered":"2024-07-15T06:50:44.804389+0000","last_clean":"2024-07-15T06:50:44.804389+0000","last_became_active":"2024-07-15T06:50:29.272992+0000","last_became_peered":"2024-07-15T06:50:29.272992+0000","last_unstale":"2024-07-15T06:50:44.804389+0000","last_undegraded":"2024-07-15T06:50:44.804389+0000","last_fullsized":"2024-07-15T06:50:44.804389+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T08:47:47.418375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":4,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.790601+0000","last_change":"2024-07-15T06:50:29.254597+0000","last_active":"2024-07-15T06:50:44.790601+0000","last_peered":"2024-07-15T06:50:44.790601+0000","last_clean":"2024-07-15T06:50:44.790601+0000","last_became_active":"2024-07-15T06:50:29.253996+0000","last_became_peered":"2024-07-15T06:50:29.253996+0000","last_unstale":"2024-07-15T06:50:44.790601+0000","last_undegraded":"2024-07-15T06:50:44.790601+0000","last_fullsized":"2024-07-15T06:50:44.790601+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T08:04:21.799515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.802216+0000","last_change":"2024-07-15T06:50:29.847314+0000","last_active":"2024-07-15T06:50:44.802216+0000","last_peered":"2024-07-15T06:50:44.802216+0000","last_clean":"2024-07-15T06:50:44.802216+0000","last_became_active":"2024-07-15T06:50:29.846456+0000","last_became_peered":"2024-07-15T06:50:29.846456+0000","last_unstale":"2024-07-15T06:50:44.802216+0000","last_undegraded":"2024-07-15T06:50:44.802216+0000","last_fullsized":"2024-07-15T06:50:44.802216+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T11:42:00.570499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.791328+0000","last_change":"2024-07-15T06:50:29.277238+0000","last_active":"2024-07-15T06:50:44.791328+0000","last_peered":"2024-07-15T06:50:44.791328+0000","last_clean":"2024-07-15T06:50:44.791328+0000","last_became_active":"2024-07-15T06:50:29.276628+0000","last_became_peered":"2024-07-15T06:50:29.276628+0000","last_unstale":"2024-07-15T06:50:44.791328+0000","last_undegraded":"2024-07-15T06:50:44.791328+0000","last_fullsized":"2024-07-15T06:50:44.791328+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T09:10:41.364615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.801890+0000","last_change":"2024-07-15T06:50:29.306634+0000","last_active":"2024-07-15T06:50:44.801890+0000","last_peered":"2024-07-15T06:50:44.801890+0000","last_clean":"2024-07-15T06:50:44.801890+0000","last_became_active":"2024-07-15T06:50:29.306345+0000","last_became_peered":"2024-07-15T06:50:29.306345+0000","last_unstale":"2024-07-15T06:50:44.801890+0000","last_undegraded":"2024-07-15T06:50:44.801890+0000","last_fullsized":"2024-07-15T06:50:44.801890+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T07:14:55.770668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.802585+0000","last_change":"2024-07-15T06:50:29.305890+0000","last_active":"2024-07-15T06:50:44.802585+0000","last_peered":"2024-07-15T06:50:44.802585+0000","last_clean":"2024-07-15T06:50:44.802585+0000","last_became_active":"2024-07-15T06:50:29.305493+0000","last_became_peered":"2024-07-15T06:50:29.305493+0000","last_unstale":"2024-07-15T06:50:44.802585+0000","last_undegraded":"2024-07-15T06:50:44.802585+0000","last_fullsized":"2024-07-15T06:50:44.802585+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T07:40:24.355598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.802157+0000","last_change":"2024-07-15T06:50:29.275180+0000","last_active":"2024-07-15T06:50:44.802157+0000","last_peered":"2024-07-15T06:50:44.802157+0000","last_clean":"2024-07-15T06:50:44.802157+0000","last_became_active":"2024-07-15T06:50:29.274633+0000","last_became_peered":"2024-07-15T06:50:29.274633+0000","last_unstale":"2024-07-15T06:50:44.802157+0000","last_undegraded":"2024-07-15T06:50:44.802157+0000","last_fullsized":"2024-07-15T06:50:44.802157+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T18:15:47.570473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.790235+0000","last_change":"2024-07-15T06:50:29.846346+0000","last_active":"2024-07-15T06:50:44.790235+0000","last_peered":"2024-07-15T06:50:44.790235+0000","last_clean":"2024-07-15T06:50:44.790235+0000","last_became_active":"2024-07-15T06:50:29.845428+0000","last_became_peered":"2024-07-15T06:50:29.845428+0000","last_unstale":"2024-07-15T06:50:44.790235+0000","last_undegraded":"2024-07-15T06:50:44.790235+0000","last_fullsized":"2024-07-15T06:50:44.790235+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T08:10:24.260944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.792398+0000","last_change":"2024-07-15T06:50:29.270851+0000","last_active":"2024-07-15T06:50:44.792398+0000","last_peered":"2024-07-15T06:50:44.792398+0000","last_clean":"2024-07-15T06:50:44.792398+0000","last_became_active":"2024-07-15T06:50:29.270552+0000","last_became_peered":"2024-07-15T06:50:29.270552+0000","last_unstale":"2024-07-15T06:50:44.792398+0000","last_undegraded":"2024-07-15T06:50:44.792398+0000","last_fullsized":"2024-07-15T06:50:44.792398+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T14:41:27.594394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.803449+0000","last_change":"2024-07-15T06:50:29.305640+0000","last_active":"2024-07-15T06:50:44.803449+0000","last_peered":"2024-07-15T06:50:44.803449+0000","last_clean":"2024-07-15T06:50:44.803449+0000","last_became_active":"2024-07-15T06:50:29.305350+0000","last_became_peered":"2024-07-15T06:50:29.305350+0000","last_unstale":"2024-07-15T06:50:44.803449+0000","last_undegraded":"2024-07-15T06:50:44.803449+0000","last_fullsized":"2024-07-15T06:50:44.803449+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T17:18:26.850757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"68'2","reported_seq":16,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.797850+0000","last_change":"2024-07-15T06:50:29.244106+0000","last_active":"2024-07-15T06:50:44.797850+0000","last_peered":"2024-07-15T06:50:44.797850+0000","last_clean":"2024-07-15T06:50:44.797850+0000","last_became_active":"2024-07-15T06:50:29.242294+0000","last_became_peered":"2024-07-15T06:50:29.242294+0000","last_unstale":"2024-07-15T06:50:44.797850+0000","last_undegraded":"2024-07-15T06:50:44.797850+0000","last_fullsized":"2024-07-15T06:50:44.797850+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T15:32:38.649353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":23,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-07-15T06:50:45.912080+0000","last_change":"2024-07-15T06:50:45.912080+0000","last_active":"2024-07-15T06:50:45.912080+0000","last_peered":"2024-07-15T06:50:45.912080+0000","last_clean":"2024-07-15T06:50:45.912080+0000","last_became_active":"2024-07-15T06:50:45.911055+0000","last_became_peered":"2024-07-15T06:50:45.911055+0000","last_unstale":"2024-07-15T06:50:45.912080+0000","last_undegraded":"2024-07-15T06:50:45.912080+0000","last_fullsized":"2024-07-15T06:50:45.912080+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T17:41:11.718209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"68'2","reported_seq":16,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.790341+0000","last_change":"2024-07-15T06:50:29.266698+0000","last_active":"2024-07-15T06:50:44.790341+0000","last_peered":"2024-07-15T06:50:44.790341+0000","last_clean":"2024-07-15T06:50:44.790341+0000","last_became_active":"2024-07-15T06:50:29.266355+0000","last_became_peered":"2024-07-15T06:50:29.266355+0000","last_unstale":"2024-07-15T06:50:44.790341+0000","last_undegraded":"2024-07-15T06:50:44.790341+0000","last_fullsized":"2024-07-15T06:50:44.790341+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T09:58:47.439567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.798794+0000","last_change":"2024-07-15T06:50:29.303449+0000","last_active":"2024-07-15T06:50:44.798794+0000","last_peered":"2024-07-15T06:50:44.798794+0000","last_clean":"2024-07-15T06:50:44.798794+0000","last_became_active":"2024-07-15T06:50:29.303028+0000","last_became_peered":"2024-07-15T06:50:29.303028+0000","last_unstale":"2024-07-15T06:50:44.798794+0000","last_undegraded":"2024-07-15T06:50:44.798794+0000","last_fullsized":"2024-07-15T06:50:44.798794+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T16:33:21.086414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":17,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.790394+0000","last_change":"2024-07-15T06:50:29.268693+0000","last_active":"2024-07-15T06:50:44.790394+0000","last_peered":"2024-07-15T06:50:44.790394+0000","last_clean":"2024-07-15T06:50:44.790394+0000","last_became_active":"2024-07-15T06:50:29.267932+0000","last_became_peered":"2024-07-15T06:50:29.267932+0000","last_unstale":"2024-07-15T06:50:44.790394+0000","last_undegraded":"2024-07-15T06:50:44.790394+0000","last_fullsized":"2024-07-15T06:50:44.790394+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T14:55:15.826736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.797649+0000","last_change":"2024-07-15T06:50:29.303795+0000","last_active":"2024-07-15T06:50:44.797649+0000","last_peered":"2024-07-15T06:50:44.797649+0000","last_clean":"2024-07-15T06:50:44.797649+0000","last_became_active":"2024-07-15T06:50:29.303252+0000","last_became_peered":"2024-07-15T06:50:29.303252+0000","last_unstale":"2024-07-15T06:50:44.797649+0000","last_undegraded":"2024-07-15T06:50:44.797649+0000","last_fullsized":"2024-07-15T06:50:44.797649+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T12:52:31.000663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":17,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.790584+0000","last_change":"2024-07-15T06:50:29.255053+0000","last_active":"2024-07-15T06:50:44.790584+0000","last_peered":"2024-07-15T06:50:44.790584+0000","last_clean":"2024-07-15T06:50:44.790584+0000","last_became_active":"2024-07-15T06:50:29.253310+0000","last_became_peered":"2024-07-15T06:50:29.253310+0000","last_unstale":"2024-07-15T06:50:44.790584+0000","last_undegraded":"2024-07-15T06:50:44.790584+0000","last_fullsized":"2024-07-15T06:50:44.790584+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T08:09:48.203498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.802395+0000","last_change":"2024-07-15T06:50:29.302910+0000","last_active":"2024-07-15T06:50:44.802395+0000","last_peered":"2024-07-15T06:50:44.802395+0000","last_clean":"2024-07-15T06:50:44.802395+0000","last_became_active":"2024-07-15T06:50:29.301648+0000","last_became_peered":"2024-07-15T06:50:29.301648+0000","last_unstale":"2024-07-15T06:50:44.802395+0000","last_undegraded":"2024-07-15T06:50:44.802395+0000","last_fullsized":"2024-07-15T06:50:44.802395+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T12:33:30.880775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'2","reported_seq":16,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.803561+0000","last_change":"2024-07-15T06:50:29.271689+0000","last_active":"2024-07-15T06:50:44.803561+0000","last_peered":"2024-07-15T06:50:44.803561+0000","last_clean":"2024-07-15T06:50:44.803561+0000","last_became_active":"2024-07-15T06:50:29.271387+0000","last_became_peered":"2024-07-15T06:50:29.271387+0000","last_unstale":"2024-07-15T06:50:44.803561+0000","last_undegraded":"2024-07-15T06:50:44.803561+0000","last_fullsized":"2024-07-15T06:50:44.803561+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T13:06:39.901886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.797211+0000","last_change":"2024-07-15T06:50:29.301207+0000","last_active":"2024-07-15T06:50:44.797211+0000","last_peered":"2024-07-15T06:50:44.797211+0000","last_clean":"2024-07-15T06:50:44.797211+0000","last_became_active":"2024-07-15T06:50:29.300787+0000","last_became_peered":"2024-07-15T06:50:29.300787+0000","last_unstale":"2024-07-15T06:50:44.797211+0000","last_undegraded":"2024-07-15T06:50:44.797211+0000","last_fullsized":"2024-07-15T06:50:44.797211+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T07:47:03.031066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"68'5","reported_seq":21,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-07-15T06:50:44.791179+0000","last_change":"2024-07-15T06:50:29.252625+0000","last_active":"2024-07-15T06:50:44.791179+0000","last_peered":"2024-07-15T06:50:44.791179+0000","last_clean":"2024-07-15T06:50:44.791179+0000","last_became_active":"2024-07-15T06:50:29.252276+0000","last_became_peered":"2024-07-15T06:50:29.252276+0000","last_unstale":"2024-07-15T06:50:44.791179+0000","last_undegraded":"2024-07-15T06:50:44.791179+0000","last_fullsized":"2024-07-15T06:50:44.791179+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T14:34:20.688321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":4,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2338,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":47,"num_read_kb":49,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":7014,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":45,"ondisk_log_size":45,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234054,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298664,"kb_used_data":1304,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446488,"statfs":{"total":95995035648,"available":95689203712,"internally_reserved":0,"allocated":1335296,"data_stored":905399,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Jul 15 06:51:55 2024","interfaces":[{"interface":"back","average":{"1min":0.94699999999999995,"5min":0.93600000000000005,"15min":0.93700000000000006},"min":{"1min":0.56699999999999995,"5min":0.443,"15min":0.443},"max":{"1min":1.1919999999999999,"5min":1.532,"15min":1.532},"last":1.123},{"interface":"front","average":{"1min":0.85199999999999998,"5min":0.88400000000000001,"15min":0.89500000000000002},"min":{"1min":0.57299999999999995,"5min":0.46600000000000003,"15min":0.46600000000000003},"max":{"1min":1.1040000000000001,"5min":2.073,"15min":2.073},"last":1.2789999999999999}]},{"osd":1,"last update":"Mon Jul 15 06:51:55 2024","interfaces":[{"interface":"back","average":{"1min":0.90200000000000002,"5min":0.90400000000000003,"15min":0.93000000000000005},"min":{"1min":0.58699999999999997,"5min":0.48499999999999999,"15min":0.48499999999999999},"max":{"1min":1.1930000000000001,"5min":1.6579999999999999,"15min":1.6579999999999999},"last":1.042},{"interface":"front","average":{"1min":0.93400000000000005,"5min":0.95599999999999996,"15min":0.99399999999999999},"min":{"1min":0.69199999999999995,"5min":0.51500000000000001,"15min":0.51500000000000001},"max":{"1min":1.1279999999999999,"5min":2.1840000000000002,"15min":2.1840000000000002},"last":0.99299999999999999}]},{"osd":2,"last update":"Mon Jul 15 06:51:55 2024","interfaces":[{"interface":"back","average":{"1min":0.873,"5min":0.91700000000000004,"15min":0.90800000000000003},"min":{"1min":0.66300000000000003,"5min":0.50900000000000001,"15min":0.50900000000000001},"max":{"1min":1.119,"5min":1.6279999999999999,"15min":1.6279999999999999},"last":1.0780000000000001},{"interface":"front","average":{"1min":1.006,"5min":0.96499999999999997,"15min":0.97899999999999998},"min":{"1min":0.64100000000000001,"5min":0.55800000000000005,"15min":0.55800000000000005},"max":{"1min":1.2030000000000001,"5min":1.802,"15min":1.802},"last":1.1359999999999999}]},{"osd":3,"last update":"Mon Jul 15 06:51:55 2024","interfaces":[{"interface":"back","average":{"1min":0.872,"5min":0.93300000000000005,"15min":0.93999999999999995},"min":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"max":{"1min":1.2310000000000001,"5min":1.704,"15min":1.704},"last":1.022},{"interface":"front","average":{"1min":1.0189999999999999,"5min":1.018,"15min":1.054},"min":{"1min":0.60599999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"max":{"1min":1.28,"5min":1.74,"15min":1.74},"last":1.232}]},{"osd":4,"last update":"Mon Jul 15 06:51:55 2024","interfaces":[{"interface":"back","average":{"1min":0.93600000000000005,"5min":0.95099999999999996,"15min":0.96699999999999997},"min":{"1min":0.69799999999999995,"5min":0.52500000000000002,"15min":0.52500000000000002},"max":{"1min":1.306,"5min":2.395,"15min":2.395},"last":0.76600000000000001},{"interface":"front","average":{"1min":1.0629999999999999,"5min":1.0249999999999999,"15min":1.0569999999999999},"min":{"1min":0.81699999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"max":{"1min":1.2729999999999999,"5min":2.129,"15min":2.129},"last":0.88600000000000001}]},{"osd":5,"last update":"Mon Jul 15 06:51:55 2024","interfaces":[{"interface":"back","average":{"1min":1.014,"5min":0.996,"15min":1.0389999999999999},"min":{"1min":0.65500000000000003,"5min":0.47999999999999998,"15min":0.47999999999999998},"max":{"1min":1.2889999999999999,"5min":2.7759999999999998,"15min":2.7759999999999998},"last":0.94099999999999995},{"interface":"front","average":{"1min":1.0660000000000001,"5min":1.0489999999999999,"15min":1.075},"min":{"1min":0.89300000000000002,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":1.3160000000000001,"5min":2.254,"15min":2.254},"last":1.1899999999999999}]},{"osd":6,"last update":"Mon Jul 15 06:51:55 2024","interfaces":[{"interface":"back","average":{"1min":1.109,"5min":1.0860000000000001,"15min":1.1200000000000001},"min":{"1min":0.94799999999999995,"5min":0.50600000000000001,"15min":0.50600000000000001},"max":{"1min":1.2709999999999999,"5min":2.3260000000000001,"15min":2.3260000000000001},"last":1.1020000000000001},{"interface":"front","average":{"1min":1.089,"5min":1.099,"15min":1.127},"min":{"1min":0.73599999999999999,"5min":0.59499999999999997,"15min":0.59499999999999997},"max":{"1min":1.292,"5min":1.893,"15min":1.893},"last":1.161}]}]},{"osd":6,"up_from":46,"seq":197568495696,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298656,"kb_used_data":1296,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446496,"statfs":{"total":95995035648,"available":95689211904,"internally_reserved":0,"allocated":1327104,"data_stored":905576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Jul 15 06:51:58 2024","interfaces":[{"interface":"back","average":{"1min":0.871,"5min":0.85299999999999998,"15min":0.90600000000000003},"min":{"1min":0.53300000000000003,"5min":0.50700000000000001,"15min":0.50700000000000001},"max":{"1min":1.1679999999999999,"5min":1.268,"15min":2.1579999999999999},"last":0.752},{"interface":"front","average":{"1min":0.90800000000000003,"5min":0.879,"15min":0.89100000000000001},"min":{"1min":0.56200000000000006,"5min":0.47099999999999997,"15min":0.47099999999999997},"max":{"1min":1.165,"5min":1.276,"15min":1.3089999999999999},"last":0.624}]},{"osd":1,"last update":"Mon Jul 15 06:51:58 2024","interfaces":[{"interface":"back","average":{"1min":0.92400000000000004,"5min":0.90800000000000003,"15min":0.91200000000000003},"min":{"1min":0.55700000000000005,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":1.1080000000000001,"5min":1.4019999999999999,"15min":1.444},"last":0.66200000000000003},{"interface":"front","average":{"1min":0.94499999999999995,"5min":0.88700000000000001,"15min":0.94899999999999995},"min":{"1min":0.623,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":1.1930000000000001,"5min":1.4119999999999999,"15min":2.2210000000000001},"last":0.88100000000000001}]},{"osd":2,"last update":"Mon Jul 15 06:51:58 2024","interfaces":[{"interface":"back","average":{"1min":0.84399999999999997,"5min":0.86299999999999999,"15min":0.95999999999999996},"min":{"1min":0.66300000000000003,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":1.327,"5min":1.327,"15min":2.754},"last":1.119},{"interface":"front","average":{"1min":0.995,"5min":0.93000000000000005,"15min":0.99399999999999999},"min":{"1min":0.65800000000000003,"5min":0.501,"15min":0.501},"max":{"1min":1.28,"5min":1.536,"15min":2.1019999999999999},"last":1.3919999999999999}]},{"osd":3,"last update":"Mon Jul 15 06:51:58 2024","interfaces":[{"interface":"back","average":{"1min":0.97099999999999997,"5min":0.93300000000000005,"15min":0.91300000000000003},"min":{"1min":0.63300000000000001,"5min":0.498,"15min":0.498},"max":{"1min":1.198,"5min":1.3799999999999999,"15min":1.3799999999999999},"last":0.81399999999999995},{"interface":"front","average":{"1min":0.97899999999999998,"5min":0.93600000000000005,"15min":0.99199999999999999},"min":{"1min":0.69099999999999995,"5min":0.52500000000000002,"15min":0.52500000000000002},"max":{"1min":1.202,"5min":1.321,"15min":2.0600000000000001},"last":1.1970000000000001}]},{"osd":4,"last update":"Mon Jul 15 06:51:58 2024","interfaces":[{"interface":"back","average":{"1min":0.84399999999999997,"5min":0.82899999999999996,"15min":0.79800000000000004},"min":{"1min":0.59599999999999997,"5min":0.38700000000000001,"15min":0.34799999999999998},"max":{"1min":1.181,"5min":1.6120000000000001,"15min":1.6120000000000001},"last":1.159},{"interface":"front","average":{"1min":0.97099999999999997,"5min":0.93200000000000005,"15min":0.872},"min":{"1min":0.66500000000000004,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":1.264,"5min":1.417,"15min":1.417},"last":0.94099999999999995}]},{"osd":5,"last update":"Mon Jul 15 06:51:58 2024","interfaces":[{"interface":"back","average":{"1min":0.93899999999999995,"5min":0.91800000000000004,"15min":0.878},"min":{"1min":0.64800000000000002,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":1.256,"5min":1.6779999999999999,"15min":1.6779999999999999},"last":1.3440000000000001},{"interface":"front","average":{"1min":0.97799999999999998,"5min":0.98599999999999999,"15min":0.89500000000000002},"min":{"1min":0.62,"5min":0.373,"15min":0.373},"max":{"1min":1.3160000000000001,"5min":1.375,"15min":1.375},"last":1.143}]},{"osd":7,"last update":"Mon Jul 15 06:51:56 2024","interfaces":[{"interface":"back","average":{"1min":1.032,"5min":1.02,"15min":0.996},"min":{"1min":0.68799999999999994,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":1.3979999999999999,"5min":1.7430000000000001,"15min":1.7430000000000001},"last":1.294},{"interface":"front","average":{"1min":1.1140000000000001,"5min":1.0349999999999999,"15min":1.0049999999999999},"min":{"1min":0.75600000000000001,"5min":0.435,"15min":0.435},"max":{"1min":1.3680000000000001,"5min":1.8540000000000001,"15min":1.8540000000000001},"last":1.2230000000000001}]}]},{"osd":5,"up_from":38,"seq":163208757338,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298064,"kb_used_data":704,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447088,"statfs":{"total":95995035648,"available":95689818112,"internally_reserved":0,"allocated":720896,"data_stored":313903,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Jul 15 06:52:10 2024","interfaces":[{"interface":"back","average":{"1min":0.99399999999999999,"5min":0.93899999999999995,"15min":0.93000000000000005},"min":{"1min":0.59099999999999997,"5min":0.47199999999999998,"15min":0.47199999999999998},"max":{"1min":1.2,"5min":1.242,"15min":1.242},"last":1.0700000000000001},{"interface":"front","average":{"1min":0.99199999999999999,"5min":0.94799999999999995,"15min":0.91100000000000003},"min":{"1min":0.70299999999999996,"5min":0.502,"15min":0.502},"max":{"1min":1.2010000000000001,"5min":1.27,"15min":1.27},"last":1.097}]},{"osd":1,"last update":"Mon Jul 15 06:52:10 2024","interfaces":[{"interface":"back","average":{"1min":0.91900000000000004,"5min":0.88400000000000001,"15min":0.83199999999999996},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":1.2010000000000001,"5min":1.2110000000000001,"15min":1.2110000000000001},"last":0.86599999999999999},{"interface":"front","average":{"1min":0.86099999999999999,"5min":0.89100000000000001,"15min":0.88700000000000001},"min":{"1min":0.65000000000000002,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":1.206,"5min":1.268,"15min":1.268},"last":0.96999999999999997}]},{"osd":2,"last update":"Mon Jul 15 06:52:10 2024","interfaces":[{"interface":"back","average":{"1min":0.877,"5min":0.873,"15min":0.83099999999999996},"min":{"1min":0.59399999999999997,"5min":0.52500000000000002,"15min":0.52500000000000002},"max":{"1min":1.194,"5min":1.2869999999999999,"15min":1.2869999999999999},"last":0.78900000000000003},{"interface":"front","average":{"1min":0.90600000000000003,"5min":0.88200000000000001,"15min":0.91000000000000003},"min":{"1min":0.55100000000000005,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":1.2709999999999999,"5min":1.2709999999999999,"15min":1.286},"last":1.2709999999999999}]},{"osd":3,"last update":"Mon Jul 15 06:52:10 2024","interfaces":[{"interface":"back","average":{"1min":0.93300000000000005,"5min":0.92700000000000005,"15min":0.86899999999999999},"min":{"1min":0.73599999999999999,"5min":0.56399999999999995,"15min":0.56399999999999995},"max":{"1min":1.2,"5min":1.2709999999999999,"15min":1.2709999999999999},"last":1.0409999999999999},{"interface":"front","average":{"1min":0.96099999999999997,"5min":0.93600000000000005,"15min":0.94399999999999995},"min":{"1min":0.70599999999999996,"5min":0.53500000000000003,"15min":0.53500000000000003},"max":{"1min":1.2310000000000001,"5min":1.256,"15min":1.3320000000000001},"last":1.01}]},{"osd":4,"last update":"Mon Jul 15 06:52:10 2024","interfaces":[{"interface":"back","average":{"1min":0.871,"5min":0.81399999999999995,"15min":0.752},"min":{"1min":0.63,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":1.0760000000000001,"5min":1.1910000000000001,"15min":1.1910000000000001},"last":0.90000000000000002},{"interface":"front","average":{"1min":0.93100000000000005,"5min":0.88300000000000001,"15min":0.79400000000000004},"min":{"1min":0.72199999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":1.1779999999999999,"5min":1.3420000000000001,"15min":1.3420000000000001},"last":0.93200000000000005}]},{"osd":6,"last update":"Mon Jul 15 06:52:04 2024","interfaces":[{"interface":"back","average":{"1min":0.96899999999999997,"5min":0.93600000000000005,"15min":0.91000000000000003},"min":{"1min":0.61599999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":1.333,"5min":1.333,"15min":1.333},"last":1.2010000000000001},{"interface":"front","average":{"1min":0.98199999999999998,"5min":0.96999999999999997,"15min":0.93300000000000005},"min":{"1min":0.63100000000000001,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":1.2869999999999999,"5min":1.3149999999999999,"15min":1.3149999999999999},"last":1.175}]},{"osd":7,"last update":"Mon Jul 15 06:51:50 2024","interfaces":[{"interface":"back","average":{"1min":1.046,"5min":1.0389999999999999,"15min":0.99299999999999999},"min":{"1min":0.77800000000000002,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":1.3600000000000001,"5min":1.3600000000000001,"15min":1.3600000000000001},"last":1.236},{"interface":"front","average":{"1min":1.046,"5min":1.026,"15min":0.998},"min":{"1min":0.85699999999999998,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":1.27,"5min":1.3939999999999999,"15min":1.3939999999999999},"last":1.1399999999999999}]}]},{"osd":4,"up_from":30,"seq":128849018981,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298084,"kb_used_data":724,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447068,"statfs":{"total":95995035648,"available":95689797632,"internally_reserved":0,"allocated":741376,"data_stored":315284,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Jul 15 06:51:15 2024","interfaces":[{"interface":"back","average":{"1min":0.82599999999999996,"5min":0.85099999999999998,"15min":0.83099999999999996},"min":{"1min":0.52400000000000002,"5min":0.46600000000000003,"15min":0.46600000000000003},"max":{"1min":1.252,"5min":1.381,"15min":1.381},"last":1.1459999999999999},{"interface":"front","average":{"1min":0.98999999999999999,"5min":0.91700000000000004,"15min":0.84699999999999998},"min":{"1min":0.70699999999999996,"5min":0.48799999999999999,"15min":0.48799999999999999},"max":{"1min":1.2589999999999999,"5min":1.381,"15min":1.381},"last":0.92000000000000004}]},{"osd":1,"last update":"Mon Jul 15 06:51:15 2024","interfaces":[{"interface":"back","average":{"1min":0.98499999999999999,"5min":0.93200000000000005,"15min":0.878},"min":{"1min":0.60999999999999999,"5min":0.497,"15min":0.497},"max":{"1min":1.24,"5min":1.3120000000000001,"15min":1.3120000000000001},"last":0.75700000000000001},{"interface":"front","average":{"1min":0.95499999999999996,"5min":0.85899999999999999,"15min":0.86899999999999999},"min":{"1min":0.56799999999999995,"5min":0.501,"15min":0.501},"max":{"1min":1.2609999999999999,"5min":1.3260000000000001,"15min":1.3260000000000001},"last":0.99399999999999999}]},{"osd":2,"last update":"Mon Jul 15 06:51:15 2024","interfaces":[{"interface":"back","average":{"1min":0.94799999999999995,"5min":0.90600000000000003,"15min":0.84899999999999998},"min":{"1min":0.70699999999999996,"5min":0.49399999999999999,"15min":0.49399999999999999},"max":{"1min":1.333,"5min":1.333,"15min":1.333},"last":0.88700000000000001},{"interface":"front","average":{"1min":0.87,"5min":0.86599999999999999,"15min":0.873},"min":{"1min":0.55300000000000005,"5min":0.52100000000000002,"15min":0.52100000000000002},"max":{"1min":1.2430000000000001,"5min":1.9910000000000001,"15min":1.9910000000000001},"last":0.94699999999999995}]},{"osd":3,"last update":"Mon Jul 15 06:51:15 2024","interfaces":[{"interface":"back","average":{"1min":0.92900000000000005,"5min":0.91100000000000003,"15min":0.874},"min":{"1min":0.59199999999999997,"5min":0.56000000000000005,"15min":0.56000000000000005},"max":{"1min":1.1579999999999999,"5min":2.1709999999999998,"15min":2.1709999999999998},"last":1.0820000000000001},{"interface":"front","average":{"1min":1.0069999999999999,"5min":0.96199999999999997,"15min":0.91600000000000004},"min":{"1min":0.66100000000000003,"5min":0.48999999999999999,"15min":0.48999999999999999},"max":{"1min":1.264,"5min":1.4970000000000001,"15min":1.4970000000000001},"last":1.0429999999999999}]},{"osd":5,"last update":"Mon Jul 15 06:52:15 2024","interfaces":[{"interface":"back","average":{"1min":0.94399999999999995,"5min":0.91000000000000003,"15min":0.77200000000000002},"min":{"1min":0.625,"5min":0.38400000000000001,"15min":0.33500000000000002},"max":{"1min":1.26,"5min":2.085,"15min":2.085},"last":0.85499999999999998},{"interface":"front","average":{"1min":0.96299999999999997,"5min":0.90100000000000002,"15min":0.76600000000000001},"min":{"1min":0.60899999999999999,"5min":0.32200000000000001,"15min":0.32000000000000001},"max":{"1min":1.2110000000000001,"5min":1.2729999999999999,"15min":1.2729999999999999},"last":1.2110000000000001}]},{"osd":6,"last update":"Mon Jul 15 06:52:01 2024","interfaces":[{"interface":"back","average":{"1min":0.98099999999999998,"5min":0.95099999999999996,"15min":0.79300000000000004},"min":{"1min":0.65200000000000002,"5min":0.35399999999999998,"15min":0.35099999999999998},"max":{"1min":1.3180000000000001,"5min":1.5760000000000001,"15min":1.5760000000000001},"last":1.248},{"interface":"front","average":{"1min":1.0169999999999999,"5min":0.96999999999999997,"15min":0.82199999999999995},"min":{"1min":0.749,"5min":0.36799999999999999,"15min":0.34000000000000002},"max":{"1min":1.2330000000000001,"5min":1.347,"15min":1.347},"last":1.121}]},{"osd":7,"last update":"Mon Jul 15 06:51:53 2024","interfaces":[{"interface":"back","average":{"1min":1.036,"5min":1.0049999999999999,"15min":0.94499999999999995},"min":{"1min":0.80200000000000005,"5min":0.378,"15min":0.378},"max":{"1min":1.264,"5min":2.2469999999999999,"15min":2.2469999999999999},"last":1.1890000000000001},{"interface":"front","average":{"1min":1.107,"5min":1.071,"15min":1.0649999999999999},"min":{"1min":0.749,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":1.3700000000000001,"5min":1.653,"15min":1.653},"last":1.3220000000000001}]}]},{"osd":3,"up_from":25,"seq":107374182512,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298060,"kb_used_data":704,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447092,"statfs":{"total":95995035648,"available":95689822208,"internally_reserved":0,"allocated":720896,"data_stored":313676,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Jul 15 06:51:24 2024","interfaces":[{"interface":"back","average":{"1min":0.64500000000000002,"5min":0.65700000000000003,"15min":0.64400000000000002},"min":{"1min":0.45600000000000002,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.90700000000000003,"5min":1.1220000000000001,"15min":1.1220000000000001},"last":0.83199999999999996},{"interface":"front","average":{"1min":0.76600000000000001,"5min":0.72299999999999998,"15min":0.70399999999999996},"min":{"1min":0.47199999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":1.056,"5min":1.647,"15min":1.647},"last":0.67200000000000004}]},{"osd":1,"last update":"Mon Jul 15 06:51:24 2024","interfaces":[{"interface":"back","average":{"1min":0.68100000000000005,"5min":0.72399999999999998,"15min":0.70699999999999996},"min":{"1min":0.54000000000000004,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.83999999999999997,"5min":1.3580000000000001,"15min":1.3580000000000001},"last":0.79100000000000004},{"interface":"front","average":{"1min":0.70299999999999996,"5min":0.72099999999999997,"15min":0.69999999999999996},"min":{"1min":0.49399999999999999,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.998,"5min":1.149,"15min":1.2190000000000001},"last":0.75900000000000001}]},{"osd":2,"last update":"Mon Jul 15 06:51:24 2024","interfaces":[{"interface":"back","average":{"1min":0.84399999999999997,"5min":0.79000000000000004,"15min":0.76200000000000001},"min":{"1min":0.56399999999999995,"5min":0.374,"15min":0.374},"max":{"1min":1.2330000000000001,"5min":1.2330000000000001,"15min":1.2330000000000001},"last":0.627},{"interface":"front","average":{"1min":0.77900000000000003,"5min":0.752,"15min":0.72699999999999998},"min":{"1min":0.59899999999999998,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":1.0369999999999999,"5min":1.3240000000000001,"15min":1.3240000000000001},"last":0.71499999999999997}]},{"osd":4,"last update":"Mon Jul 15 06:51:18 2024","interfaces":[{"interface":"back","average":{"1min":1.006,"5min":0.94599999999999995,"15min":0.89000000000000001},"min":{"1min":0.70099999999999996,"5min":0.59599999999999997,"15min":0.59599999999999997},"max":{"1min":1.198,"5min":1.2949999999999999,"15min":1.2949999999999999},"last":0.95299999999999996},{"interface":"front","average":{"1min":0.97099999999999997,"5min":0.95799999999999996,"15min":0.91200000000000003},"min":{"1min":0.76900000000000002,"5min":0.57299999999999995,"15min":0.57299999999999995},"max":{"1min":1.1719999999999999,"5min":1.8140000000000001,"15min":1.8140000000000001},"last":1.0309999999999999}]},{"osd":5,"last update":"Mon Jul 15 06:51:15 2024","interfaces":[{"interface":"back","average":{"1min":1.0369999999999999,"5min":0.995,"15min":0.96999999999999997},"min":{"1min":0.79900000000000004,"5min":0.57299999999999995,"15min":0.57299999999999995},"max":{"1min":1.3500000000000001,"5min":1.7609999999999999,"15min":1.7609999999999999},"last":0.98199999999999998},{"interface":"front","average":{"1min":0.97499999999999998,"5min":0.95899999999999996,"15min":0.97899999999999998},"min":{"1min":0.81599999999999995,"5min":0.64900000000000002,"15min":0.64900000000000002},"max":{"1min":1.1439999999999999,"5min":1.429,"15min":1.5069999999999999},"last":1.0029999999999999}]},{"osd":6,"last update":"Mon Jul 15 06:52:07 2024","interfaces":[{"interface":"back","average":{"1min":1.042,"5min":1.018,"15min":1.069},"min":{"1min":0.66100000000000003,"5min":0.65600000000000003,"15min":0.64500000000000002},"max":{"1min":1.542,"5min":1.5880000000000001,"15min":1.8899999999999999},"last":0.89400000000000002},{"interface":"front","average":{"1min":1.0640000000000001,"5min":0.996,"15min":1.0129999999999999},"min":{"1min":0.82999999999999996,"5min":0.61499999999999999,"15min":0.61499999999999999},"max":{"1min":1.6120000000000001,"5min":1.6120000000000001,"15min":1.6120000000000001},"last":0.92200000000000004}]},{"osd":7,"last update":"Mon Jul 15 06:51:57 2024","interfaces":[{"interface":"back","average":{"1min":1.206,"5min":1.054,"15min":1.034},"min":{"1min":0.80900000000000005,"5min":0.58299999999999996,"15min":0.58299999999999996},"max":{"1min":2.6960000000000002,"5min":2.6960000000000002,"15min":2.6960000000000002},"last":0.85899999999999999},{"interface":"front","average":{"1min":1.103,"5min":1.056,"15min":1.0900000000000001},"min":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"max":{"1min":1.7509999999999999,"5min":1.7509999999999999,"15min":1.7509999999999999},"last":1.1499999999999999}]}]},{"osd":2,"up_from":18,"seq":77309411449,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298064,"kb_used_data":708,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447088,"statfs":{"total":95995035648,"available":95689818112,"internally_reserved":0,"allocated":724992,"data_stored":314190,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Jul 15 06:51:42 2024","interfaces":[{"interface":"back","average":{"1min":0.58499999999999996,"5min":0.623,"15min":0.59599999999999997},"min":{"1min":0.30399999999999999,"5min":0.28299999999999997,"15min":0.22700000000000001},"max":{"1min":0.77300000000000002,"5min":1.4690000000000001,"15min":1.4690000000000001},"last":0.58399999999999996},{"interface":"front","average":{"1min":0.68100000000000005,"5min":0.68000000000000005,"15min":0.61699999999999999},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":1.0840000000000001,"5min":1.0880000000000001,"15min":1.095},"last":0.53600000000000003}]},{"osd":1,"last update":"Mon Jul 15 06:51:42 2024","interfaces":[{"interface":"back","average":{"1min":0.66300000000000003,"5min":0.70699999999999996,"15min":0.65900000000000003},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":1.0940000000000001,"5min":1.3149999999999999,"15min":1.3149999999999999},"last":0.80700000000000005},{"interface":"front","average":{"1min":0.753,"5min":0.75700000000000001,"15min":0.67700000000000005},"min":{"1min":0.215,"5min":0.215,"15min":0.215},"max":{"1min":1.1779999999999999,"5min":1.2330000000000001,"15min":1.2330000000000001},"last":0.77100000000000002}]},{"osd":3,"last update":"Mon Jul 15 06:51:31 2024","interfaces":[{"interface":"back","average":{"1min":0.76000000000000001,"5min":0.76100000000000001,"15min":0.75700000000000001},"min":{"1min":0.35699999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":1.0600000000000001,"5min":1.256,"15min":1.256},"last":1.0469999999999999},{"interface":"front","average":{"1min":0.80700000000000005,"5min":0.82199999999999995,"15min":0.76000000000000001},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":1.133,"5min":1.264,"15min":1.264},"last":0.94299999999999995}]},{"osd":4,"last update":"Mon Jul 15 06:51:14 2024","interfaces":[{"interface":"back","average":{"1min":1.077,"5min":0.98899999999999999,"15min":0.91400000000000003},"min":{"1min":0.83699999999999997,"5min":0.56299999999999994,"15min":0.56299999999999994},"max":{"1min":1.5129999999999999,"5min":1.5129999999999999,"15min":1.5129999999999999},"last":0.99199999999999999},{"interface":"front","average":{"1min":1.121,"5min":1.0569999999999999,"15min":0.97099999999999997},"min":{"1min":0.89700000000000002,"5min":0.50800000000000001,"15min":0.50800000000000001},"max":{"1min":1.3520000000000001,"5min":1.5589999999999999,"15min":1.5589999999999999},"last":1.0629999999999999}]},{"osd":5,"last update":"Mon Jul 15 06:51:14 2024","interfaces":[{"interface":"back","average":{"1min":1.0409999999999999,"5min":1.0229999999999999,"15min":0.97799999999999998},"min":{"1min":0.72799999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"max":{"1min":1.444,"5min":1.7290000000000001,"15min":1.7290000000000001},"last":1.0169999999999999},{"interface":"front","average":{"1min":1.028,"5min":0.98099999999999998,"15min":0.96399999999999997},"min":{"1min":0.70199999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"max":{"1min":1.401,"5min":1.571,"15min":1.571},"last":1.091}]},{"osd":6,"last update":"Mon Jul 15 06:52:06 2024","interfaces":[{"interface":"back","average":{"1min":0.98999999999999999,"5min":1.038,"15min":1.0600000000000001},"min":{"1min":0.53300000000000003,"5min":0.52800000000000002,"15min":0.52800000000000002},"max":{"1min":1.349,"5min":1.8009999999999999,"15min":1.8009999999999999},"last":0.83899999999999997},{"interface":"front","average":{"1min":0.91000000000000003,"5min":0.94199999999999995,"15min":0.95099999999999996},"min":{"1min":0.64000000000000001,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":1.141,"5min":2.3500000000000001,"15min":2.3500000000000001},"last":0.86699999999999999}]},{"osd":7,"last update":"Mon Jul 15 06:51:54 2024","interfaces":[{"interface":"back","average":{"1min":1.02,"5min":1.038,"15min":1.0589999999999999},"min":{"1min":0.67600000000000005,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":1.468,"5min":1.6299999999999999,"15min":1.6299999999999999},"last":1.127},{"interface":"front","average":{"1min":1.022,"5min":1.087,"15min":1.075},"min":{"1min":0.63800000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"max":{"1min":1.5469999999999999,"5min":1.756,"15min":1.756},"last":1.113}]}]},{"osd":1,"up_from":13,"seq":55834574980,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298056,"kb_used_data":700,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447096,"statfs":{"total":95995035648,"available":95689826304,"internally_reserved":0,"allocated":716800,"data_stored":313586,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Jul 15 06:51:49 2024","interfaces":[{"interface":"back","average":{"1min":0.63600000000000001,"5min":0.65400000000000003,"15min":0.59099999999999997},"min":{"1min":0.38400000000000001,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.96899999999999997,"5min":1.2250000000000001,"15min":1.2250000000000001},"last":0.60599999999999998},{"interface":"front","average":{"1min":0.70699999999999996,"5min":0.71599999999999997,"15min":0.63100000000000001},"min":{"1min":0.39600000000000002,"5min":0.32700000000000001,"15min":0.312},"max":{"1min":1.1479999999999999,"5min":1.2709999999999999,"15min":1.5609999999999999},"last":0.71199999999999997}]},{"osd":2,"last update":"Mon Jul 15 06:51:30 2024","interfaces":[{"interface":"back","average":{"1min":0.68400000000000005,"5min":0.749,"15min":0.68600000000000005},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.28299999999999997},"max":{"1min":1.1339999999999999,"5min":1.4379999999999999,"15min":1.7509999999999999},"last":0.81499999999999995},{"interface":"front","average":{"1min":0.71999999999999997,"5min":0.74299999999999999,"15min":0.64900000000000002},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.29599999999999999},"max":{"1min":1.0740000000000001,"5min":1.47,"15min":1.47},"last":1.2370000000000001}]},{"osd":3,"last update":"Mon Jul 15 06:51:30 2024","interfaces":[{"interface":"back","average":{"1min":0.77400000000000002,"5min":0.79800000000000004,"15min":0.78000000000000003},"min":{"1min":0.372,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":1.0700000000000001,"5min":1.3220000000000001,"15min":1.3220000000000001},"last":1.002},{"interface":"front","average":{"1min":0.71599999999999997,"5min":0.75600000000000001,"15min":0.72499999999999998},"min":{"1min":0.41999999999999998,"5min":0.377,"15min":0.377},"max":{"1min":1.044,"5min":1.431,"15min":1.431},"last":0.77500000000000002}]},{"osd":4,"last update":"Mon Jul 15 06:51:17 2024","interfaces":[{"interface":"back","average":{"1min":0.99299999999999999,"5min":1.03,"15min":0.95699999999999996},"min":{"1min":0.68799999999999994,"5min":0.66200000000000003,"15min":0.66200000000000003},"max":{"1min":1.226,"5min":1.5840000000000001,"15min":1.5840000000000001},"last":0.89300000000000002},{"interface":"front","average":{"1min":0.96599999999999997,"5min":0.96999999999999997,"15min":0.92800000000000005},"min":{"1min":0.747,"5min":0.60299999999999998,"15min":0.60299999999999998},"max":{"1min":1.21,"5min":1.6839999999999999,"15min":1.6839999999999999},"last":1.101}]},{"osd":5,"last update":"Mon Jul 15 06:52:09 2024","interfaces":[{"interface":"back","average":{"1min":0.94799999999999995,"5min":0.96599999999999997,"15min":1.036},"min":{"1min":0.69399999999999995,"5min":0.63,"15min":0.63},"max":{"1min":1.375,"5min":1.5109999999999999,"15min":4.431},"last":1.387},{"interface":"front","average":{"1min":0.94399999999999995,"5min":0.98299999999999998,"15min":1.022},"min":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"max":{"1min":1.173,"5min":1.4630000000000001,"15min":3.0670000000000002},"last":1.492}]},{"osd":6,"last update":"Mon Jul 15 06:51:59 2024","interfaces":[{"interface":"back","average":{"1min":1.012,"5min":1.034,"15min":1.0620000000000001},"min":{"1min":0.624,"5min":0.624,"15min":0.624},"max":{"1min":1.5449999999999999,"5min":1.5620000000000001,"15min":2.0339999999999998},"last":1.1919999999999999},{"interface":"front","average":{"1min":1.0640000000000001,"5min":1.0389999999999999,"15min":1.081},"min":{"1min":0.66400000000000003,"5min":0.65200000000000002,"15min":0.65200000000000002},"max":{"1min":1.3500000000000001,"5min":1.617,"15min":1.897},"last":1.4039999999999999}]},{"osd":7,"last update":"Mon Jul 15 06:51:57 2024","interfaces":[{"interface":"back","average":{"1min":1.0549999999999999,"5min":1.0960000000000001,"15min":1.1299999999999999},"min":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"max":{"1min":1.6140000000000001,"5min":1.988,"15min":1.988},"last":1.056},{"interface":"front","average":{"1min":1.0629999999999999,"5min":1.0389999999999999,"15min":1.0609999999999999},"min":{"1min":0.64600000000000002,"5min":0.64200000000000002,"15min":0.64200000000000002},"max":{"1min":1.5329999999999999,"5min":1.7809999999999999,"15min":1.7809999999999999},"last":1.294}]}]},{"osd":0,"up_from":8,"seq":34359738511,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298644,"kb_used_data":1288,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446508,"statfs":{"total":95995035648,"available":95689224192,"internally_reserved":0,"allocated":1318912,"data_stored":904546,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Jul 15 06:51:42 2024","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":0.65200000000000002,"15min":0.58999999999999997},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.26900000000000002},"max":{"1min":0.875,"5min":1.0469999999999999,"15min":1.2070000000000001},"last":0.53500000000000003},{"interface":"front","average":{"1min":0.629,"5min":0.71999999999999997,"15min":0.64300000000000002},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":1.071,"5min":1.236,"15min":1.236},"last":0.72399999999999998}]},{"osd":2,"last update":"Mon Jul 15 06:51:32 2024","interfaces":[{"interface":"back","average":{"1min":0.72599999999999998,"5min":0.755,"15min":0.69099999999999995},"min":{"1min":0.435,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":1.1759999999999999,"5min":1.294,"15min":1.294},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.71099999999999997,"5min":0.70899999999999996,"15min":0.67700000000000005},"min":{"1min":0.48299999999999998,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.98599999999999999,"5min":1.54,"15min":1.54},"last":0.97899999999999998}]},{"osd":3,"last update":"Mon Jul 15 06:51:24 2024","interfaces":[{"interface":"back","average":{"1min":0.84699999999999998,"5min":0.79900000000000004,"15min":0.77400000000000002},"min":{"1min":0.45900000000000002,"5min":0.371,"15min":0.371},"max":{"1min":1.252,"5min":1.7470000000000001,"15min":1.7470000000000001},"last":0.76100000000000001},{"interface":"front","average":{"1min":0.70399999999999996,"5min":0.73399999999999999,"15min":0.71699999999999997},"min":{"1min":0.40300000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":1.0489999999999999,"5min":1.0489999999999999,"15min":1.0489999999999999},"last":0.81399999999999995}]},{"osd":4,"last update":"Mon Jul 15 06:51:19 2024","interfaces":[{"interface":"back","average":{"1min":0.94199999999999995,"5min":0.95299999999999996,"15min":0.90700000000000003},"min":{"1min":0.746,"5min":0.51800000000000002,"15min":0.51800000000000002},"max":{"1min":1.2050000000000001,"5min":1.2909999999999999,"15min":1.2909999999999999},"last":0.93999999999999995},{"interface":"front","average":{"1min":1.0369999999999999,"5min":1.018,"15min":0.94699999999999995},"min":{"1min":0.69199999999999995,"5min":0.52800000000000002,"15min":0.52800000000000002},"max":{"1min":1.518,"5min":1.518,"15min":1.518},"last":1.1579999999999999}]},{"osd":5,"last update":"Mon Jul 15 06:52:04 2024","interfaces":[{"interface":"back","average":{"1min":0.89500000000000002,"5min":0.91800000000000004,"15min":0.94099999999999995},"min":{"1min":0.59299999999999997,"5min":0.58399999999999996,"15min":0.58399999999999996},"max":{"1min":1.26,"5min":1.3049999999999999,"15min":1.3280000000000001},"last":0.91800000000000004},{"interface":"front","average":{"1min":0.93899999999999995,"5min":0.96199999999999997,"15min":0.95699999999999996},"min":{"1min":0.69099999999999995,"5min":0.51000000000000001,"15min":0.51000000000000001},"max":{"1min":1.4099999999999999,"5min":1.619,"15min":1.619},"last":1.2569999999999999}]},{"osd":6,"last update":"Mon Jul 15 06:52:04 2024","interfaces":[{"interface":"back","average":{"1min":0.95899999999999996,"5min":0.996,"15min":0.98099999999999998},"min":{"1min":0.57799999999999996,"5min":0.5,"15min":0.5},"max":{"1min":1.3740000000000001,"5min":1.3740000000000001,"15min":1.3740000000000001},"last":0.85499999999999998},{"interface":"front","average":{"1min":0.98299999999999998,"5min":1.0129999999999999,"15min":1.0369999999999999},"min":{"1min":0.621,"5min":0.47799999999999998,"15min":0.47799999999999998},"max":{"1min":1.5669999999999999,"5min":1.5669999999999999,"15min":1.5669999999999999},"last":1.212}]},{"osd":7,"last update":"Mon Jul 15 06:51:53 2024","interfaces":[{"interface":"back","average":{"1min":0.996,"5min":1.048,"15min":1.048},"min":{"1min":0.67400000000000004,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":1.4650000000000001,"5min":1.4650000000000001,"15min":1.4650000000000001},"last":1.0129999999999999},{"interface":"front","average":{"1min":0.998,"5min":1.0600000000000001,"15min":1.052},"min":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"max":{"1min":1.518,"5min":1.827,"15min":1.827},"last":1.3049999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1698,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1622,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-07-15T06:52:20.152 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-07-15T06:52:20.152 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-07-15T06:52:20.152 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-07-15T06:52:20.152 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph health --format=json 2024-07-15T06:52:20.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:52:20 smithi134 bash[24068]: audit 2024-07-15T06:52:19.370422+0000 mgr.y (mgr.24272) 178 : audit [DBG] from='client.14715 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-07-15T06:52:20.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:52:20 smithi089 bash[21793]: audit 2024-07-15T06:52:19.370422+0000 mgr.y (mgr.24272) 178 : audit [DBG] from='client.14715 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-07-15T06:52:20.619 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:52:20 smithi089 bash[31372]: audit 2024-07-15T06:52:19.370422+0000 mgr.y (mgr.24272) 178 : audit [DBG] from='client.14715 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-07-15T06:52:21.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:52:21 smithi134 bash[24068]: cluster 2024-07-15T06:52:20.192275+0000 mgr.y (mgr.24272) 179 : cluster [DBG] pgmap v152: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:52:21.619 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:52:21 smithi089 bash[31372]: cluster 2024-07-15T06:52:20.192275+0000 mgr.y (mgr.24272) 179 : cluster [DBG] pgmap v152: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:52:21.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:52:21 smithi089 bash[21793]: cluster 2024-07-15T06:52:20.192275+0000 mgr.y (mgr.24272) 179 : cluster [DBG] pgmap v152: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:52:23.119 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:52:22 smithi089 bash[22045]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:52:22] "GET /metrics HTTP/1.1" 200 239257 "" "Prometheus/2.43.0" 2024-07-15T06:52:23.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:52:23 smithi134 bash[24068]: cluster 2024-07-15T06:52:22.192802+0000 mgr.y (mgr.24272) 180 : cluster [DBG] pgmap v153: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:52:23.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:52:23 smithi089 bash[21793]: cluster 2024-07-15T06:52:22.192802+0000 mgr.y (mgr.24272) 180 : cluster [DBG] pgmap v153: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:52:23.619 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:52:23 smithi089 bash[31372]: cluster 2024-07-15T06:52:22.192802+0000 mgr.y (mgr.24272) 180 : cluster [DBG] pgmap v153: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:52:24.926 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T06:52:25.369 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:52:25 smithi089 bash[21793]: cluster 2024-07-15T06:52:24.193405+0000 mgr.y (mgr.24272) 181 : cluster [DBG] pgmap v154: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:52:25.369 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:52:25 smithi089 bash[31372]: cluster 2024-07-15T06:52:24.193405+0000 mgr.y (mgr.24272) 181 : cluster [DBG] pgmap v154: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:52:25.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:52:25 smithi134 bash[24068]: cluster 2024-07-15T06:52:24.193405+0000 mgr.y (mgr.24272) 181 : cluster [DBG] pgmap v154: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:52:27.263 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:52:26 smithi134 bash[25019]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:52:26] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-07-15T06:52:27.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:52:27 smithi134 bash[24068]: cluster 2024-07-15T06:52:26.194176+0000 mgr.y (mgr.24272) 182 : cluster [DBG] pgmap v155: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:52:27.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:52:27 smithi089 bash[21793]: cluster 2024-07-15T06:52:26.194176+0000 mgr.y (mgr.24272) 182 : cluster [DBG] pgmap v155: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:52:27.619 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:52:27 smithi089 bash[31372]: cluster 2024-07-15T06:52:26.194176+0000 mgr.y (mgr.24272) 182 : cluster [DBG] pgmap v155: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:52:28.436 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-07-15T06:52:28.436 INFO:teuthology.orchestra.run.smithi089.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-07-15T06:52:29.180 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-07-15T06:52:29.181 INFO:teuthology.run_tasks:Running task print... 2024-07-15T06:52:29.191 INFO:teuthology.task.print:**** upgrading first half of cluster, with stress **** 2024-07-15T06:52:29.191 INFO:teuthology.run_tasks:Running task parallel... 2024-07-15T06:52:29.206 INFO:teuthology.task.parallel:starting parallel... 2024-07-15T06:52:29.206 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-07-15T06:52:29.207 INFO:teuthology.task.sequential:In sequential, running task workunit... 2024-07-15T06:52:29.211 INFO:tasks.workunit:Pulling workunits from ref quincy 2024-07-15T06:52:29.212 INFO:tasks.workunit:Making a separate scratch dir for every client... 2024-07-15T06:52:29.212 DEBUG:teuthology.orchestra.run.smithi089:> stat -- /home/ubuntu/cephtest/mnt.0 2024-07-15T06:52:29.212 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-07-15T06:52:29.213 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-07-15T06:52:29.213 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi089.front.sepia.ceph.com 2024-07-15T06:52:29.213 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -e sha1=6ea7aaa9f93d34dc4d21862afe00140dfddbd706 -- bash -c 'ceph config set mgr mgr/cephadm/daemon_cache_timeout 60' 2024-07-15T06:52:29.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-07-15T06:52:29.219 INFO:teuthology.orchestra.run.smithi089.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2024-07-15T06:52:29.219 DEBUG:teuthology.orchestra.run.smithi089:> mkdir -- /home/ubuntu/cephtest/mnt.0 2024-07-15T06:52:29.270 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2024-07-15T06:52:29.270 DEBUG:teuthology.orchestra.run.smithi089:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2024-07-15T06:52:29.321 INFO:tasks.workunit:timeout=3h 2024-07-15T06:52:29.321 INFO:tasks.workunit:cleanup=True 2024-07-15T06:52:29.321 DEBUG:teuthology.orchestra.run.smithi089:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch quincy https://git.ceph.com/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2024-07-15T06:52:29.374 INFO:tasks.workunit.client.0.smithi089.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2024-07-15T06:52:29.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:52:29 smithi134 bash[24068]: cluster 2024-07-15T06:52:28.194579+0000 mgr.y (mgr.24272) 183 : cluster [DBG] pgmap v156: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:52:29.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:52:29 smithi134 bash[24068]: audit 2024-07-15T06:52:28.441840+0000 mon.a (mon.0) 897 : audit [DBG] from='client.? 172.21.15.89:0/3250743465' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-15T06:52:29.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:52:29 smithi134 bash[24068]: audit 2024-07-15T06:52:29.194694+0000 mon.a (mon.0) 898 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:52:29.619 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:52:29 smithi089 bash[31372]: cluster 2024-07-15T06:52:28.194579+0000 mgr.y (mgr.24272) 183 : cluster [DBG] pgmap v156: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:52:29.619 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:52:29 smithi089 bash[31372]: audit 2024-07-15T06:52:28.441840+0000 mon.a (mon.0) 897 : audit [DBG] from='client.? 172.21.15.89:0/3250743465' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-15T06:52:29.619 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:52:29 smithi089 bash[31372]: audit 2024-07-15T06:52:29.194694+0000 mon.a (mon.0) 898 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:52:29.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:52:29 smithi089 bash[21793]: cluster 2024-07-15T06:52:28.194579+0000 mgr.y (mgr.24272) 183 : cluster [DBG] pgmap v156: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:52:29.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:52:29 smithi089 bash[21793]: audit 2024-07-15T06:52:28.441840+0000 mon.a (mon.0) 897 : audit [DBG] from='client.? 172.21.15.89:0/3250743465' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-15T06:52:29.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:52:29 smithi089 bash[21793]: audit 2024-07-15T06:52:29.194694+0000 mon.a (mon.0) 898 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:52:31.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:52:31 smithi134 bash[24068]: cluster 2024-07-15T06:52:30.195574+0000 mgr.y (mgr.24272) 184 : cluster [DBG] pgmap v157: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:52:31.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:52:31 smithi089 bash[21793]: cluster 2024-07-15T06:52:30.195574+0000 mgr.y (mgr.24272) 184 : cluster [DBG] pgmap v157: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:52:31.619 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:52:31 smithi089 bash[31372]: cluster 2024-07-15T06:52:30.195574+0000 mgr.y (mgr.24272) 184 : cluster [DBG] pgmap v157: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:52:33.119 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:52:32 smithi089 bash[22045]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:52:32] "GET /metrics HTTP/1.1" 200 239257 "" "Prometheus/2.43.0" 2024-07-15T06:52:33.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:52:33 smithi134 bash[24068]: cluster 2024-07-15T06:52:32.196162+0000 mgr.y (mgr.24272) 185 : cluster [DBG] pgmap v158: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:52:33.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:52:33 smithi089 bash[21793]: cluster 2024-07-15T06:52:32.196162+0000 mgr.y (mgr.24272) 185 : cluster [DBG] pgmap v158: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:52:33.619 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:52:33 smithi089 bash[31372]: cluster 2024-07-15T06:52:32.196162+0000 mgr.y (mgr.24272) 185 : cluster [DBG] pgmap v158: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:52:33.999 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T06:52:34.697 DEBUG:teuthology.orchestra.run.smithi089:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2024-07-15T06:52:34.746 INFO:tasks.workunit.client.0.smithi089.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2024-07-15T06:52:34.748 INFO:tasks.workunit.client.0.smithi089.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-07-15T06:52:34.748 INFO:tasks.workunit.client.0.smithi089.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2024-07-15T06:52:35.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:52:35 smithi134 bash[24068]: cluster 2024-07-15T06:52:34.196754+0000 mgr.y (mgr.24272) 186 : cluster [DBG] pgmap v159: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:52:35.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:52:35 smithi089 bash[21793]: cluster 2024-07-15T06:52:34.196754+0000 mgr.y (mgr.24272) 186 : cluster [DBG] pgmap v159: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:52:35.619 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:52:35 smithi089 bash[31372]: cluster 2024-07-15T06:52:34.196754+0000 mgr.y (mgr.24272) 186 : cluster [DBG] pgmap v159: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:52:37.282 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:52:36 smithi134 bash[25019]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:52:36] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-07-15T06:52:37.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:52:37 smithi134 bash[24068]: cluster 2024-07-15T06:52:36.197793+0000 mgr.y (mgr.24272) 187 : cluster [DBG] pgmap v160: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:52:37.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:52:37 smithi089 bash[21793]: cluster 2024-07-15T06:52:36.197793+0000 mgr.y (mgr.24272) 187 : cluster [DBG] pgmap v160: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:52:37.619 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:52:37 smithi089 bash[31372]: cluster 2024-07-15T06:52:36.197793+0000 mgr.y (mgr.24272) 187 : cluster [DBG] pgmap v160: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:52:39.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:52:39 smithi134 bash[24068]: cluster 2024-07-15T06:52:38.198393+0000 mgr.y (mgr.24272) 188 : cluster [DBG] pgmap v161: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:52:39.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:52:39 smithi089 bash[21793]: cluster 2024-07-15T06:52:38.198393+0000 mgr.y (mgr.24272) 188 : cluster [DBG] pgmap v161: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:52:39.619 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:52:39 smithi089 bash[31372]: cluster 2024-07-15T06:52:38.198393+0000 mgr.y (mgr.24272) 188 : cluster [DBG] pgmap v161: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:52:39.718 INFO:tasks.workunit.client.0.smithi089.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2024-07-15T06:52:39.784 INFO:tasks.workunit.client.0.smithi089.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2024-07-15T06:52:39.842 INFO:tasks.workunit.client.0.smithi089.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-07-15T06:52:39.842 INFO:tasks.workunit.client.0.smithi089.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-07-15T06:52:39.842 INFO:tasks.workunit.client.0.smithi089.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2024-07-15T06:52:39.886 INFO:tasks.workunit.client.0.smithi089.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-07-15T06:52:39.889 DEBUG:teuthology.orchestra.run.smithi089:> set -ex 2024-07-15T06:52:39.889 DEBUG:teuthology.orchestra.run.smithi089:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2024-07-15T06:52:39.940 INFO:tasks.workunit:Running workunits matching cls/test_cls_rbd.sh on client.0... 2024-07-15T06:52:39.942 INFO:tasks.workunit:Running workunit cls/test_cls_rbd.sh... 2024-07-15T06:52:39.942 DEBUG:teuthology.orchestra.run.smithi089:workunit test cls/test_cls_rbd.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh 2024-07-15T06:52:39.998 INFO:tasks.workunit.client.0.smithi089.stdout:Running main() from gmock_main.cc 2024-07-15T06:52:39.998 INFO:tasks.workunit.client.0.smithi089.stdout:[==========] Running 70 tests from 1 test suite. 2024-07-15T06:52:39.998 INFO:tasks.workunit.client.0.smithi089.stdout:[----------] Global test environment set-up. 2024-07-15T06:52:39.998 INFO:tasks.workunit.client.0.smithi089.stdout:[----------] 70 tests from TestClsRbd 2024-07-15T06:52:40.694 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:52:40 smithi089 bash[22045]: [15/Jul/2024:06:52:40] ENGINE Bus STOPPING 2024-07-15T06:52:40.695 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:52:40 smithi089 bash[22045]: [15/Jul/2024:06:52:40] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-07-15T06:52:40.695 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:52:40 smithi089 bash[22045]: [15/Jul/2024:06:52:40] ENGINE Bus STOPPED 2024-07-15T06:52:40.695 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:52:40 smithi089 bash[22045]: [15/Jul/2024:06:52:40] ENGINE Bus STARTING 2024-07-15T06:52:41.119 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:52:40 smithi089 bash[22045]: [15/Jul/2024:06:52:40] ENGINE Serving on http://:::9283 2024-07-15T06:52:41.119 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:52:40 smithi089 bash[22045]: [15/Jul/2024:06:52:40] ENGINE Bus STARTED 2024-07-15T06:52:41.456 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -e sha1=6ea7aaa9f93d34dc4d21862afe00140dfddbd706 -- bash -c 'ceph config set global log_to_journald false --force' 2024-07-15T06:52:41.491 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.get_all_features 2024-07-15T06:52:41.529 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.get_all_features (38 ms) 2024-07-15T06:52:41.529 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.copyup 2024-07-15T06:52:41.765 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.copyup (236 ms) 2024-07-15T06:52:41.765 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.sparse_copyup 2024-07-15T06:52:41.793 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.sparse_copyup (28 ms) 2024-07-15T06:52:41.793 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.get_and_set_id 2024-07-15T06:52:41.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:52:41 smithi134 bash[24068]: cluster 2024-07-15T06:52:40.199417+0000 mgr.y (mgr.24272) 189 : cluster [DBG] pgmap v162: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:52:41.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:52:41 smithi134 bash[24068]: audit 2024-07-15T06:52:40.437768+0000 mon.a (mon.0) 899 : audit [INF] from='client.? ' entity='client.admin' 2024-07-15T06:52:41.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:52:41 smithi134 bash[24068]: audit 2024-07-15T06:52:40.447423+0000 mon.a (mon.0) 900 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:52:41.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:52:41 smithi134 bash[24068]: cluster 2024-07-15T06:52:40.484428+0000 mon.a (mon.0) 901 : cluster [DBG] osdmap e71: 8 total, 8 up, 8 in 2024-07-15T06:52:41.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:52:41 smithi134 bash[24068]: audit 2024-07-15T06:52:40.492156+0000 mon.a (mon.0) 902 : audit [INF] from='client.? 172.21.15.89:0/1874034267' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi089-68574-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-15T06:52:41.810 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.get_and_set_id (18 ms) 2024-07-15T06:52:41.810 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.add_remove_child 2024-07-15T06:52:41.837 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.add_remove_child (26 ms) 2024-07-15T06:52:41.837 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.directory_methods 2024-07-15T06:52:41.869 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:52:41 smithi089 bash[31372]: cluster 2024-07-15T06:52:40.199417+0000 mgr.y (mgr.24272) 189 : cluster [DBG] pgmap v162: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:52:41.869 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:52:41 smithi089 bash[31372]: audit 2024-07-15T06:52:40.437768+0000 mon.a (mon.0) 899 : audit [INF] from='client.? ' entity='client.admin' 2024-07-15T06:52:41.869 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:52:41 smithi089 bash[31372]: audit 2024-07-15T06:52:40.447423+0000 mon.a (mon.0) 900 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:52:41.869 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:52:41 smithi089 bash[31372]: cluster 2024-07-15T06:52:40.484428+0000 mon.a (mon.0) 901 : cluster [DBG] osdmap e71: 8 total, 8 up, 8 in 2024-07-15T06:52:41.869 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:52:41 smithi089 bash[31372]: audit 2024-07-15T06:52:40.492156+0000 mon.a (mon.0) 902 : audit [INF] from='client.? 172.21.15.89:0/1874034267' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi089-68574-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-15T06:52:41.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:52:41 smithi089 bash[21793]: cluster 2024-07-15T06:52:40.199417+0000 mgr.y (mgr.24272) 189 : cluster [DBG] pgmap v162: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:52:41.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:52:41 smithi089 bash[21793]: audit 2024-07-15T06:52:40.437768+0000 mon.a (mon.0) 899 : audit [INF] from='client.? ' entity='client.admin' 2024-07-15T06:52:41.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:52:41 smithi089 bash[21793]: audit 2024-07-15T06:52:40.447423+0000 mon.a (mon.0) 900 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:52:41.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:52:41 smithi089 bash[21793]: cluster 2024-07-15T06:52:40.484428+0000 mon.a (mon.0) 901 : cluster [DBG] osdmap e71: 8 total, 8 up, 8 in 2024-07-15T06:52:41.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:52:41 smithi089 bash[21793]: audit 2024-07-15T06:52:40.492156+0000 mon.a (mon.0) 902 : audit [INF] from='client.? 172.21.15.89:0/1874034267' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi089-68574-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-15T06:52:41.908 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.directory_methods (71 ms) 2024-07-15T06:52:41.908 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.create 2024-07-15T06:52:41.943 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.create (35 ms) 2024-07-15T06:52:41.943 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.get_features 2024-07-15T06:52:41.947 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.get_features (4 ms) 2024-07-15T06:52:41.947 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.get_object_prefix 2024-07-15T06:52:41.951 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.get_object_prefix (4 ms) 2024-07-15T06:52:41.951 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.get_create_timestamp 2024-07-15T06:52:41.956 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.get_create_timestamp (5 ms) 2024-07-15T06:52:41.956 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.get_access_timestamp 2024-07-15T06:52:41.960 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.get_access_timestamp (4 ms) 2024-07-15T06:52:41.960 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.get_modify_timestamp 2024-07-15T06:52:41.962 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.get_modify_timestamp (3 ms) 2024-07-15T06:52:41.963 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.get_data_pool 2024-07-15T06:52:41.972 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.get_data_pool (10 ms) 2024-07-15T06:52:41.972 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.get_size 2024-07-15T06:52:41.982 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.get_size (10 ms) 2024-07-15T06:52:41.983 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.set_size 2024-07-15T06:52:41.994 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.set_size (12 ms) 2024-07-15T06:52:41.994 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.protection_status 2024-07-15T06:52:42.042 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.protection_status (47 ms) 2024-07-15T06:52:42.042 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.snapshot_limits 2024-07-15T06:52:42.060 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.snapshot_limits (19 ms) 2024-07-15T06:52:42.060 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.parents_v1 2024-07-15T06:52:42.172 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.parents_v1 (111 ms) 2024-07-15T06:52:42.172 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.parents_v2 2024-07-15T06:52:42.248 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.parents_v2 (76 ms) 2024-07-15T06:52:42.248 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.snapshots 2024-07-15T06:52:42.310 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.snapshots (63 ms) 2024-07-15T06:52:42.311 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.snapid_race 2024-07-15T06:52:42.323 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.snapid_race (12 ms) 2024-07-15T06:52:42.323 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.stripingv2 2024-07-15T06:52:42.347 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.stripingv2 (25 ms) 2024-07-15T06:52:42.348 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.object_map_save 2024-07-15T06:52:42.352 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.object_map_save (4 ms) 2024-07-15T06:52:42.352 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.object_map_resize 2024-07-15T06:52:42.364 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.object_map_resize (13 ms) 2024-07-15T06:52:42.365 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.object_map_update 2024-07-15T06:52:42.377 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.object_map_update (12 ms) 2024-07-15T06:52:42.377 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.object_map_load_enoent 2024-07-15T06:52:42.377 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.object_map_load_enoent (1 ms) 2024-07-15T06:52:42.378 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.object_map_snap_add 2024-07-15T06:52:42.388 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.object_map_snap_add (10 ms) 2024-07-15T06:52:42.388 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.object_map_snap_remove 2024-07-15T06:52:42.402 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.object_map_snap_remove (15 ms) 2024-07-15T06:52:42.403 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.flags 2024-07-15T06:52:42.417 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.flags (14 ms) 2024-07-15T06:52:42.417 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.metadata 2024-07-15T06:52:42.484 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.metadata (67 ms) 2024-07-15T06:52:42.484 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.set_features 2024-07-15T06:52:42.506 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.set_features (23 ms) 2024-07-15T06:52:42.507 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.mirror 2024-07-15T06:52:42.596 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.mirror (89 ms) 2024-07-15T06:52:42.597 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.mirror_image 2024-07-15T06:52:42.634 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.mirror_image (38 ms) 2024-07-15T06:52:42.634 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.mirror_image_status 2024-07-15T06:52:42.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:52:42 smithi134 bash[24068]: audit 2024-07-15T06:52:41.491340+0000 mon.a (mon.0) 903 : audit [INF] from='client.? 172.21.15.89:0/1874034267' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi089-68574-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-15T06:52:42.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:52:42 smithi134 bash[24068]: cluster 2024-07-15T06:52:41.491578+0000 mon.a (mon.0) 904 : cluster [DBG] osdmap e72: 8 total, 8 up, 8 in 2024-07-15T06:52:42.869 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:52:42 smithi089 bash[22045]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:52:42] "GET /metrics HTTP/1.1" 200 239257 "" "Prometheus/2.43.0" 2024-07-15T06:52:42.869 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:52:42 smithi089 bash[21793]: audit 2024-07-15T06:52:41.491340+0000 mon.a (mon.0) 903 : audit [INF] from='client.? 172.21.15.89:0/1874034267' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi089-68574-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-15T06:52:42.869 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:52:42 smithi089 bash[21793]: cluster 2024-07-15T06:52:41.491578+0000 mon.a (mon.0) 904 : cluster [DBG] osdmap e72: 8 total, 8 up, 8 in 2024-07-15T06:52:42.869 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:52:42 smithi089 bash[31372]: audit 2024-07-15T06:52:41.491340+0000 mon.a (mon.0) 903 : audit [INF] from='client.? 172.21.15.89:0/1874034267' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi089-68574-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-15T06:52:42.870 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:52:42 smithi089 bash[31372]: cluster 2024-07-15T06:52:41.491578+0000 mon.a (mon.0) 904 : cluster [DBG] osdmap e72: 8 total, 8 up, 8 in 2024-07-15T06:52:43.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:52:43 smithi134 bash[24068]: cluster 2024-07-15T06:52:42.200225+0000 mgr.y (mgr.24272) 190 : cluster [DBG] pgmap v165: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:52:43.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:52:43 smithi134 bash[24068]: cluster 2024-07-15T06:52:42.493222+0000 mon.a (mon.0) 905 : cluster [DBG] osdmap e73: 8 total, 8 up, 8 in 2024-07-15T06:52:43.869 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:52:43 smithi089 bash[21793]: cluster 2024-07-15T06:52:42.200225+0000 mgr.y (mgr.24272) 190 : cluster [DBG] pgmap v165: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:52:43.869 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:52:43 smithi089 bash[21793]: cluster 2024-07-15T06:52:42.493222+0000 mon.a (mon.0) 905 : cluster [DBG] osdmap e73: 8 total, 8 up, 8 in 2024-07-15T06:52:43.870 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:52:43 smithi089 bash[31372]: cluster 2024-07-15T06:52:42.200225+0000 mgr.y (mgr.24272) 190 : cluster [DBG] pgmap v165: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:52:43.870 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:52:43 smithi089 bash[31372]: cluster 2024-07-15T06:52:42.493222+0000 mon.a (mon.0) 905 : cluster [DBG] osdmap e73: 8 total, 8 up, 8 in 2024-07-15T06:52:44.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:52:44 smithi134 bash[24068]: audit 2024-07-15T06:52:44.195367+0000 mon.a (mon.0) 906 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:52:44.869 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:52:44 smithi089 bash[31372]: audit 2024-07-15T06:52:44.195367+0000 mon.a (mon.0) 906 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:52:44.869 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:52:44 smithi089 bash[21793]: audit 2024-07-15T06:52:44.195367+0000 mon.a (mon.0) 906 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:52:45.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:52:45 smithi134 bash[24068]: cluster 2024-07-15T06:52:44.204016+0000 mgr.y (mgr.24272) 191 : cluster [DBG] pgmap v167: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:52:45.869 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:52:45 smithi089 bash[21793]: cluster 2024-07-15T06:52:44.204016+0000 mgr.y (mgr.24272) 191 : cluster [DBG] pgmap v167: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:52:45.869 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:52:45 smithi089 bash[31372]: cluster 2024-07-15T06:52:44.204016+0000 mgr.y (mgr.24272) 191 : cluster [DBG] pgmap v167: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:52:46.272 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T06:52:46.602 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.mirror_image_status (3965 ms) 2024-07-15T06:52:46.602 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.mirror_image_map 2024-07-15T06:52:46.792 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.mirror_image_map (193 ms) 2024-07-15T06:52:46.792 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.mirror_instances 2024-07-15T06:52:46.811 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.mirror_instances (19 ms) 2024-07-15T06:52:46.811 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.mirror_snapshot 2024-07-15T06:52:46.841 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.mirror_snapshot (31 ms) 2024-07-15T06:52:46.842 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.group_dir_list 2024-07-15T06:52:46.847 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.group_dir_list (6 ms) 2024-07-15T06:52:46.848 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.group_dir_add 2024-07-15T06:52:46.852 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.group_dir_add (5 ms) 2024-07-15T06:52:46.853 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.dir_add_already_existing 2024-07-15T06:52:46.861 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.dir_add_already_existing (9 ms) 2024-07-15T06:52:46.861 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.group_dir_rename 2024-07-15T06:52:46.877 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.group_dir_rename (15 ms) 2024-07-15T06:52:46.877 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.group_dir_remove 2024-07-15T06:52:46.885 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.group_dir_remove (9 ms) 2024-07-15T06:52:46.886 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.group_dir_remove_missing 2024-07-15T06:52:46.897 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.group_dir_remove_missing (11 ms) 2024-07-15T06:52:46.897 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.group_image_add 2024-07-15T06:52:46.902 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.group_image_add (5 ms) 2024-07-15T06:52:46.902 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.group_image_remove 2024-07-15T06:52:46.910 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.group_image_remove (8 ms) 2024-07-15T06:52:46.910 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.group_image_list 2024-07-15T06:52:46.917 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.group_image_list (7 ms) 2024-07-15T06:52:46.917 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.group_image_clean 2024-07-15T06:52:46.926 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.group_image_clean (10 ms) 2024-07-15T06:52:46.926 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.image_group_add 2024-07-15T06:52:46.932 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.image_group_add (5 ms) 2024-07-15T06:52:46.932 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.image_group_remove 2024-07-15T06:52:46.940 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.image_group_remove (8 ms) 2024-07-15T06:52:46.940 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.image_group_get 2024-07-15T06:52:46.946 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.image_group_get (6 ms) 2024-07-15T06:52:46.946 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_name 2024-07-15T06:52:46.950 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.group_snap_set_empty_name (4 ms) 2024-07-15T06:52:46.950 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_id 2024-07-15T06:52:46.955 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.group_snap_set_empty_id (5 ms) 2024-07-15T06:52:46.955 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_id 2024-07-15T06:52:47.218 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_id (263 ms) 2024-07-15T06:52:47.218 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_name 2024-07-15T06:52:47.225 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_name (8 ms) 2024-07-15T06:52:47.226 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.group_snap_set 2024-07-15T06:52:47.231 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.group_snap_set (5 ms) 2024-07-15T06:52:47.231 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.group_snap_list 2024-07-15T06:52:47.239 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.group_snap_list (8 ms) 2024-07-15T06:52:47.239 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.group_snap_list_max_return 2024-07-15T06:52:47.290 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.group_snap_list_max_return (51 ms) 2024-07-15T06:52:47.290 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.group_snap_list_max_read 2024-07-15T06:52:47.304 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:52:46 smithi134 bash[25019]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:52:46] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-07-15T06:52:47.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:52:46 smithi134 bash[24068]: audit 2024-07-15T06:52:45.861726+0000 mon.a (mon.0) 907 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:52:47.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:52:46 smithi134 bash[24068]: audit 2024-07-15T06:52:45.866385+0000 mon.a (mon.0) 908 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:52:47.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:52:46 smithi134 bash[24068]: audit 2024-07-15T06:52:46.124137+0000 mon.a (mon.0) 909 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:52:47.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:52:46 smithi134 bash[24068]: audit 2024-07-15T06:52:46.129534+0000 mon.a (mon.0) 910 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:52:47.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:52:46 smithi134 bash[24068]: cluster 2024-07-15T06:52:46.204661+0000 mgr.y (mgr.24272) 192 : cluster [DBG] pgmap v168: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.8 MiB/s rd, 982 KiB/s wr, 817 op/s 2024-07-15T06:52:47.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:52:46 smithi134 bash[24068]: audit 2024-07-15T06:52:46.498636+0000 mon.a (mon.0) 911 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:52:47.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:52:46 smithi134 bash[24068]: audit 2024-07-15T06:52:46.499698+0000 mon.a (mon.0) 912 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:52:47.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:52:46 smithi134 bash[24068]: audit 2024-07-15T06:52:46.505716+0000 mon.a (mon.0) 913 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:52:47.369 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:52:46 smithi089 bash[21793]: audit 2024-07-15T06:52:45.861726+0000 mon.a (mon.0) 907 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:52:47.369 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:52:46 smithi089 bash[21793]: audit 2024-07-15T06:52:45.866385+0000 mon.a (mon.0) 908 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:52:47.369 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:52:46 smithi089 bash[21793]: audit 2024-07-15T06:52:46.124137+0000 mon.a (mon.0) 909 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:52:47.369 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:52:46 smithi089 bash[21793]: audit 2024-07-15T06:52:46.129534+0000 mon.a (mon.0) 910 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:52:47.369 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:52:46 smithi089 bash[21793]: cluster 2024-07-15T06:52:46.204661+0000 mgr.y (mgr.24272) 192 : cluster [DBG] pgmap v168: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.8 MiB/s rd, 982 KiB/s wr, 817 op/s 2024-07-15T06:52:47.369 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:52:46 smithi089 bash[21793]: audit 2024-07-15T06:52:46.498636+0000 mon.a (mon.0) 911 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:52:47.369 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:52:46 smithi089 bash[21793]: audit 2024-07-15T06:52:46.499698+0000 mon.a (mon.0) 912 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:52:47.369 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:52:46 smithi089 bash[21793]: audit 2024-07-15T06:52:46.505716+0000 mon.a (mon.0) 913 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:52:47.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:52:46 smithi089 bash[31372]: audit 2024-07-15T06:52:45.861726+0000 mon.a (mon.0) 907 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:52:47.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:52:46 smithi089 bash[31372]: audit 2024-07-15T06:52:45.866385+0000 mon.a (mon.0) 908 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:52:47.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:52:46 smithi089 bash[31372]: audit 2024-07-15T06:52:46.124137+0000 mon.a (mon.0) 909 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:52:47.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:52:46 smithi089 bash[31372]: audit 2024-07-15T06:52:46.129534+0000 mon.a (mon.0) 910 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:52:47.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:52:46 smithi089 bash[31372]: cluster 2024-07-15T06:52:46.204661+0000 mgr.y (mgr.24272) 192 : cluster [DBG] pgmap v168: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.8 MiB/s rd, 982 KiB/s wr, 817 op/s 2024-07-15T06:52:47.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:52:46 smithi089 bash[31372]: audit 2024-07-15T06:52:46.498636+0000 mon.a (mon.0) 911 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:52:47.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:52:46 smithi089 bash[31372]: audit 2024-07-15T06:52:46.499698+0000 mon.a (mon.0) 912 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:52:47.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:52:46 smithi089 bash[31372]: audit 2024-07-15T06:52:46.505716+0000 mon.a (mon.0) 913 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:52:47.728 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.group_snap_list_max_read (439 ms) 2024-07-15T06:52:47.729 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.group_snap_remove 2024-07-15T06:52:47.737 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.group_snap_remove (8 ms) 2024-07-15T06:52:47.737 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.group_snap_get_by_id 2024-07-15T06:52:47.742 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.group_snap_get_by_id (5 ms) 2024-07-15T06:52:47.742 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.trash_methods 2024-07-15T06:52:47.759 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.trash_methods (18 ms) 2024-07-15T06:52:47.760 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.op_features 2024-07-15T06:52:47.775 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.op_features (16 ms) 2024-07-15T06:52:47.776 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.clone_parent 2024-07-15T06:52:47.828 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.clone_parent (52 ms) 2024-07-15T06:52:47.828 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.clone_parent_ns 2024-07-15T06:52:47.853 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.clone_parent_ns (25 ms) 2024-07-15T06:52:47.853 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.clone_child 2024-07-15T06:52:47.883 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.clone_child (30 ms) 2024-07-15T06:52:47.883 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.namespace_methods 2024-07-15T06:52:47.901 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.namespace_methods (18 ms) 2024-07-15T06:52:47.901 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.migration 2024-07-15T06:52:47.931 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.migration (30 ms) 2024-07-15T06:52:47.931 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.migration_v1 2024-07-15T06:52:47.952 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.migration_v1 (22 ms) 2024-07-15T06:52:47.952 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.assert_snapc_seq 2024-07-15T06:52:49.144 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.assert_snapc_seq (1191 ms) 2024-07-15T06:52:49.144 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.sparsify 2024-07-15T06:52:49.211 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.sparsify (67 ms) 2024-07-15T06:52:49.369 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:52:49 smithi089 bash[21793]: cluster 2024-07-15T06:52:48.137379+0000 mon.a (mon.0) 914 : cluster [DBG] osdmap e74: 8 total, 8 up, 8 in 2024-07-15T06:52:49.369 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:52:49 smithi089 bash[21793]: cluster 2024-07-15T06:52:48.205292+0000 mgr.y (mgr.24272) 193 : cluster [DBG] pgmap v170: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.6 MiB/s rd, 878 KiB/s wr, 730 op/s 2024-07-15T06:52:49.369 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:52:49 smithi089 bash[31372]: cluster 2024-07-15T06:52:48.137379+0000 mon.a (mon.0) 914 : cluster [DBG] osdmap e74: 8 total, 8 up, 8 in 2024-07-15T06:52:49.369 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:52:49 smithi089 bash[31372]: cluster 2024-07-15T06:52:48.205292+0000 mgr.y (mgr.24272) 193 : cluster [DBG] pgmap v170: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.6 MiB/s rd, 878 KiB/s wr, 730 op/s 2024-07-15T06:52:49.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:52:49 smithi134 bash[24068]: cluster 2024-07-15T06:52:48.137379+0000 mon.a (mon.0) 914 : cluster [DBG] osdmap e74: 8 total, 8 up, 8 in 2024-07-15T06:52:49.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:52:49 smithi134 bash[24068]: cluster 2024-07-15T06:52:48.205292+0000 mgr.y (mgr.24272) 193 : cluster [DBG] pgmap v170: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.6 MiB/s rd, 878 KiB/s wr, 730 op/s 2024-07-15T06:52:50.158 INFO:tasks.workunit.client.0.smithi089.stdout:[----------] 70 tests from TestClsRbd (7720 ms total) 2024-07-15T06:52:50.158 INFO:tasks.workunit.client.0.smithi089.stdout: 2024-07-15T06:52:50.158 INFO:tasks.workunit.client.0.smithi089.stdout:[----------] Global test environment tear-down 2024-07-15T06:52:50.158 INFO:tasks.workunit.client.0.smithi089.stdout:[==========] 70 tests from 1 test suite ran. (10152 ms total) 2024-07-15T06:52:50.159 INFO:tasks.workunit.client.0.smithi089.stdout:[ PASSED ] 70 tests. 2024-07-15T06:52:50.160 INFO:teuthology.orchestra.run:Running command with timeout 3600 2024-07-15T06:52:50.160 DEBUG:teuthology.orchestra.run.smithi089:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-07-15T06:52:50.210 INFO:tasks.workunit:Stopping ['cls/test_cls_rbd.sh'] on client.0... 2024-07-15T06:52:50.210 DEBUG:teuthology.orchestra.run.smithi089:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2024-07-15T06:52:50.476 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:52:50 smithi089 bash[21793]: cluster 2024-07-15T06:52:49.149109+0000 mon.a (mon.0) 915 : cluster [DBG] osdmap e75: 8 total, 8 up, 8 in 2024-07-15T06:52:50.476 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:52:50 smithi089 bash[31372]: cluster 2024-07-15T06:52:49.149109+0000 mon.a (mon.0) 915 : cluster [DBG] osdmap e75: 8 total, 8 up, 8 in 2024-07-15T06:52:50.477 DEBUG:teuthology.parallel:result is None 2024-07-15T06:52:50.478 DEBUG:teuthology.orchestra.run.smithi089:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2024-07-15T06:52:50.489 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2024-07-15T06:52:50.489 DEBUG:teuthology.orchestra.run.smithi089:> rmdir -- /home/ubuntu/cephtest/mnt.0 2024-07-15T06:52:50.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:52:50 smithi134 bash[24068]: cluster 2024-07-15T06:52:49.149109+0000 mon.a (mon.0) 915 : cluster [DBG] osdmap e75: 8 total, 8 up, 8 in 2024-07-15T06:52:50.560 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2024-07-15T06:52:50.560 INFO:teuthology.task.sequential:In sequential, running task print... 2024-07-15T06:52:50.561 INFO:teuthology.task.print:**** done cls/test_cls_rbd.sh 5-workload 2024-07-15T06:52:50.561 DEBUG:teuthology.parallel:result is None 2024-07-15T06:52:51.370 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -e sha1=6ea7aaa9f93d34dc4d21862afe00140dfddbd706 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-07-15T06:52:51.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:52:51 smithi134 bash[24068]: cluster 2024-07-15T06:52:50.151904+0000 mon.a (mon.0) 916 : cluster [DBG] osdmap e76: 8 total, 8 up, 8 in 2024-07-15T06:52:51.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:52:51 smithi134 bash[24068]: cluster 2024-07-15T06:52:50.205812+0000 mgr.y (mgr.24272) 194 : cluster [DBG] pgmap v173: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:52:51.619 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:52:51 smithi089 bash[31372]: cluster 2024-07-15T06:52:50.151904+0000 mon.a (mon.0) 916 : cluster [DBG] osdmap e76: 8 total, 8 up, 8 in 2024-07-15T06:52:51.619 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:52:51 smithi089 bash[31372]: cluster 2024-07-15T06:52:50.205812+0000 mgr.y (mgr.24272) 194 : cluster [DBG] pgmap v173: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:52:51.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:52:51 smithi089 bash[21793]: cluster 2024-07-15T06:52:50.151904+0000 mon.a (mon.0) 916 : cluster [DBG] osdmap e76: 8 total, 8 up, 8 in 2024-07-15T06:52:51.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:52:51 smithi089 bash[21793]: cluster 2024-07-15T06:52:50.205812+0000 mgr.y (mgr.24272) 194 : cluster [DBG] pgmap v173: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:52:53.119 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:52:52 smithi089 bash[22045]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:52:52] "GET /metrics HTTP/1.1" 200 241534 "" "Prometheus/2.43.0" 2024-07-15T06:52:53.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:52:53 smithi134 bash[24068]: cluster 2024-07-15T06:52:52.206376+0000 mgr.y (mgr.24272) 195 : cluster [DBG] pgmap v174: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:52:53.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:52:53 smithi089 bash[21793]: cluster 2024-07-15T06:52:52.206376+0000 mgr.y (mgr.24272) 195 : cluster [DBG] pgmap v174: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:52:53.619 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:52:53 smithi089 bash[31372]: cluster 2024-07-15T06:52:52.206376+0000 mgr.y (mgr.24272) 195 : cluster [DBG] pgmap v174: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:52:55.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:52:55 smithi134 bash[24068]: cluster 2024-07-15T06:52:54.206989+0000 mgr.y (mgr.24272) 196 : cluster [DBG] pgmap v175: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:52:55.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:52:55 smithi089 bash[21793]: cluster 2024-07-15T06:52:54.206989+0000 mgr.y (mgr.24272) 196 : cluster [DBG] pgmap v175: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:52:55.619 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:52:55 smithi089 bash[31372]: cluster 2024-07-15T06:52:54.206989+0000 mgr.y (mgr.24272) 196 : cluster [DBG] pgmap v175: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:52:56.157 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T06:52:57.265 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:52:56 smithi134 bash[25019]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:52:56] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-07-15T06:52:57.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:52:57 smithi134 bash[24068]: cluster 2024-07-15T06:52:56.207942+0000 mgr.y (mgr.24272) 197 : cluster [DBG] pgmap v176: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:52:57.582 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:52:57 smithi089 bash[21793]: cluster 2024-07-15T06:52:56.207942+0000 mgr.y (mgr.24272) 197 : cluster [DBG] pgmap v176: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:52:57.583 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:52:57 smithi089 bash[31372]: cluster 2024-07-15T06:52:56.207942+0000 mgr.y (mgr.24272) 197 : cluster [DBG] pgmap v176: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:52:59.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:52:59 smithi134 bash[24068]: cluster 2024-07-15T06:52:58.208547+0000 mgr.y (mgr.24272) 198 : cluster [DBG] pgmap v177: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:52:59.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:52:59 smithi134 bash[24068]: audit 2024-07-15T06:52:59.195477+0000 mon.a (mon.0) 917 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:52:59.619 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:52:59 smithi089 bash[31372]: cluster 2024-07-15T06:52:58.208547+0000 mgr.y (mgr.24272) 198 : cluster [DBG] pgmap v177: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:52:59.619 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:52:59 smithi089 bash[31372]: audit 2024-07-15T06:52:59.195477+0000 mon.a (mon.0) 917 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:52:59.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:52:59 smithi089 bash[21793]: cluster 2024-07-15T06:52:58.208547+0000 mgr.y (mgr.24272) 198 : cluster [DBG] pgmap v177: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:52:59.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:52:59 smithi089 bash[21793]: audit 2024-07-15T06:52:59.195477+0000 mon.a (mon.0) 917 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:52:59.780 INFO:teuthology.orchestra.run.smithi089.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:6ea7aaa9f93d34dc4d21862afe00140dfddbd706 2024-07-15T06:53:00.693 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -e sha1=6ea7aaa9f93d34dc4d21862afe00140dfddbd706 -- bash -c 'ceph orch ps' 2024-07-15T06:53:01.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:53:00 smithi134 bash[24068]: audit 2024-07-15T06:52:59.777409+0000 mgr.y (mgr.24272) 199 : audit [DBG] from='client.24656 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:6ea7aaa9f93d34dc4d21862afe00140dfddbd706", "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T06:53:01.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:53:00 smithi134 bash[24068]: cephadm 2024-07-15T06:52:59.778507+0000 mgr.y (mgr.24272) 200 : cephadm [INF] Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:6ea7aaa9f93d34dc4d21862afe00140dfddbd706 2024-07-15T06:53:01.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:53:00 smithi134 bash[24068]: audit 2024-07-15T06:52:59.785770+0000 mon.a (mon.0) 918 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:53:01.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:53:00 smithi134 bash[24068]: audit 2024-07-15T06:52:59.786728+0000 mon.a (mon.0) 919 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:53:01.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:53:00 smithi134 bash[24068]: audit 2024-07-15T06:52:59.789508+0000 mon.a (mon.0) 920 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:53:01.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:53:00 smithi134 bash[24068]: audit 2024-07-15T06:52:59.790454+0000 mon.a (mon.0) 921 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:53:01.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:53:00 smithi134 bash[24068]: audit 2024-07-15T06:52:59.797546+0000 mon.a (mon.0) 922 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:53:01.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:53:00 smithi089 bash[21793]: audit 2024-07-15T06:52:59.777409+0000 mgr.y (mgr.24272) 199 : audit [DBG] from='client.24656 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:6ea7aaa9f93d34dc4d21862afe00140dfddbd706", "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T06:53:01.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:53:00 smithi089 bash[21793]: cephadm 2024-07-15T06:52:59.778507+0000 mgr.y (mgr.24272) 200 : cephadm [INF] Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:6ea7aaa9f93d34dc4d21862afe00140dfddbd706 2024-07-15T06:53:01.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:53:00 smithi089 bash[21793]: audit 2024-07-15T06:52:59.785770+0000 mon.a (mon.0) 918 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:53:01.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:53:00 smithi089 bash[21793]: audit 2024-07-15T06:52:59.786728+0000 mon.a (mon.0) 919 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:53:01.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:53:00 smithi089 bash[21793]: audit 2024-07-15T06:52:59.789508+0000 mon.a (mon.0) 920 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:53:01.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:53:00 smithi089 bash[21793]: audit 2024-07-15T06:52:59.790454+0000 mon.a (mon.0) 921 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:53:01.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:53:00 smithi089 bash[21793]: audit 2024-07-15T06:52:59.797546+0000 mon.a (mon.0) 922 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:53:01.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:53:00 smithi089 bash[31372]: audit 2024-07-15T06:52:59.777409+0000 mgr.y (mgr.24272) 199 : audit [DBG] from='client.24656 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:6ea7aaa9f93d34dc4d21862afe00140dfddbd706", "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T06:53:01.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:53:00 smithi089 bash[31372]: cephadm 2024-07-15T06:52:59.778507+0000 mgr.y (mgr.24272) 200 : cephadm [INF] Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:6ea7aaa9f93d34dc4d21862afe00140dfddbd706 2024-07-15T06:53:01.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:53:00 smithi089 bash[31372]: audit 2024-07-15T06:52:59.785770+0000 mon.a (mon.0) 918 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:53:01.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:53:00 smithi089 bash[31372]: audit 2024-07-15T06:52:59.786728+0000 mon.a (mon.0) 919 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:53:01.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:53:00 smithi089 bash[31372]: audit 2024-07-15T06:52:59.789508+0000 mon.a (mon.0) 920 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:53:01.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:53:00 smithi089 bash[31372]: audit 2024-07-15T06:52:59.790454+0000 mon.a (mon.0) 921 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:53:01.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:53:00 smithi089 bash[31372]: audit 2024-07-15T06:52:59.797546+0000 mon.a (mon.0) 922 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:53:02.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:53:01 smithi134 bash[24068]: cephadm 2024-07-15T06:52:59.842024+0000 mgr.y (mgr.24272) 201 : cephadm [INF] Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:6ea7aaa9f93d34dc4d21862afe00140dfddbd706 2024-07-15T06:53:02.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:53:01 smithi134 bash[24068]: cluster 2024-07-15T06:53:00.209576+0000 mgr.y (mgr.24272) 202 : cluster [DBG] pgmap v178: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:53:02.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:53:01 smithi089 bash[21793]: cephadm 2024-07-15T06:52:59.842024+0000 mgr.y (mgr.24272) 201 : cephadm [INF] Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:6ea7aaa9f93d34dc4d21862afe00140dfddbd706 2024-07-15T06:53:02.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:53:01 smithi089 bash[21793]: cluster 2024-07-15T06:53:00.209576+0000 mgr.y (mgr.24272) 202 : cluster [DBG] pgmap v178: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:53:02.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:53:01 smithi089 bash[31372]: cephadm 2024-07-15T06:52:59.842024+0000 mgr.y (mgr.24272) 201 : cephadm [INF] Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:6ea7aaa9f93d34dc4d21862afe00140dfddbd706 2024-07-15T06:53:02.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:53:01 smithi089 bash[31372]: cluster 2024-07-15T06:53:00.209576+0000 mgr.y (mgr.24272) 202 : cluster [DBG] pgmap v178: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:53:03.119 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:53:02 smithi089 bash[22045]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:53:02] "GET /metrics HTTP/1.1" 200 239985 "" "Prometheus/2.43.0" 2024-07-15T06:53:04.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:53:03 smithi134 bash[24068]: cluster 2024-07-15T06:53:02.210254+0000 mgr.y (mgr.24272) 203 : cluster [DBG] pgmap v179: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:53:04.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:53:03 smithi089 bash[21793]: cluster 2024-07-15T06:53:02.210254+0000 mgr.y (mgr.24272) 203 : cluster [DBG] pgmap v179: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:53:04.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:53:03 smithi089 bash[31372]: cluster 2024-07-15T06:53:02.210254+0000 mgr.y (mgr.24272) 203 : cluster [DBG] pgmap v179: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:53:05.484 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T06:53:06.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:53:05 smithi134 bash[24068]: cluster 2024-07-15T06:53:04.210917+0000 mgr.y (mgr.24272) 204 : cluster [DBG] pgmap v180: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:53:06.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:53:05 smithi089 bash[21793]: cluster 2024-07-15T06:53:04.210917+0000 mgr.y (mgr.24272) 204 : cluster [DBG] pgmap v180: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:53:06.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:53:05 smithi089 bash[31372]: cluster 2024-07-15T06:53:04.210917+0000 mgr.y (mgr.24272) 204 : cluster [DBG] pgmap v180: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:53:07.304 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:53:06 smithi134 bash[25019]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:53:06] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-07-15T06:53:07.869 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:53:07 smithi089 bash[21793]: cluster 2024-07-15T06:53:06.212122+0000 mgr.y (mgr.24272) 205 : cluster [DBG] pgmap v181: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:53:07.869 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:53:07 smithi089 bash[31372]: cluster 2024-07-15T06:53:06.212122+0000 mgr.y (mgr.24272) 205 : cluster [DBG] pgmap v181: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:53:08.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:53:07 smithi134 bash[24068]: cluster 2024-07-15T06:53:06.212122+0000 mgr.y (mgr.24272) 205 : cluster [DBG] pgmap v181: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:53:10.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:53:09 smithi089 bash[21793]: cluster 2024-07-15T06:53:08.212644+0000 mgr.y (mgr.24272) 206 : cluster [DBG] pgmap v182: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:53:10.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:53:09 smithi089 bash[31372]: cluster 2024-07-15T06:53:08.212644+0000 mgr.y (mgr.24272) 206 : cluster [DBG] pgmap v182: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:53:10.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:53:09 smithi134 bash[24068]: cluster 2024-07-15T06:53:08.212644+0000 mgr.y (mgr.24272) 206 : cluster [DBG] pgmap v182: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:53:12.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:53:11 smithi089 bash[21793]: cluster 2024-07-15T06:53:10.213719+0000 mgr.y (mgr.24272) 207 : cluster [DBG] pgmap v183: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:53:12.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:53:11 smithi089 bash[31372]: cluster 2024-07-15T06:53:10.213719+0000 mgr.y (mgr.24272) 207 : cluster [DBG] pgmap v183: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:53:12.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:53:11 smithi134 bash[24068]: cluster 2024-07-15T06:53:10.213719+0000 mgr.y (mgr.24272) 207 : cluster [DBG] pgmap v183: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:53:12.392 INFO:teuthology.orchestra.run.smithi089.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-07-15T06:53:12.392 INFO:teuthology.orchestra.run.smithi089.stdout:alertmanager.a smithi089 *:9093,9094 running (4m) 26s ago 5m 13.8M - 0.25.0 c8568f914cd2 42ee628573b3 2024-07-15T06:53:12.392 INFO:teuthology.orchestra.run.smithi089.stdout:grafana.a smithi134 *:3000 running (4m) 26s ago 4m 80.8M - 9.4.7 954c08fa6188 f394c704b661 2024-07-15T06:53:12.392 INFO:teuthology.orchestra.run.smithi089.stdout:mds.foo.smithi089.cczuoz smithi089 running (2m) 26s ago 2m 11.6M - 17.2.7-1102-g9a202024 280b03ed6869 a1f028849336 2024-07-15T06:53:12.393 INFO:teuthology.orchestra.run.smithi089.stdout:mds.foo.smithi134.wugtzh smithi134 running (2m) 26s ago 2m 13.0M - 17.2.7-1102-g9a202024 280b03ed6869 fb5bbeafa32b 2024-07-15T06:53:12.393 INFO:teuthology.orchestra.run.smithi089.stdout:mgr.x smithi134 *:8443 running (13m) 26s ago 13m 388M - 17.2.7-1102-g9a202024 280b03ed6869 ae5dfc11e005 2024-07-15T06:53:12.393 INFO:teuthology.orchestra.run.smithi089.stdout:mgr.y smithi089 *:9283 running (15m) 26s ago 15m 440M - 17.2.7-1102-g9a202024 280b03ed6869 ac60b3a13a85 2024-07-15T06:53:12.393 INFO:teuthology.orchestra.run.smithi089.stdout:mon.a smithi089 running (15m) 26s ago 16m 49.4M 2048M 17.2.7-1102-g9a202024 280b03ed6869 40a92d660143 2024-07-15T06:53:12.393 INFO:teuthology.orchestra.run.smithi089.stdout:mon.b smithi134 running (13m) 26s ago 13m 38.7M 2048M 17.2.7-1102-g9a202024 280b03ed6869 19182ed25bde 2024-07-15T06:53:12.393 INFO:teuthology.orchestra.run.smithi089.stdout:mon.c smithi089 running (13m) 26s ago 13m 38.2M 2048M 17.2.7-1102-g9a202024 280b03ed6869 ad83089d3cbf 2024-07-15T06:53:12.393 INFO:teuthology.orchestra.run.smithi089.stdout:node-exporter.a smithi089 *:9100 running (5m) 26s ago 5m 8047k - 1.5.0 0da6a335fe13 7bca94b61f28 2024-07-15T06:53:12.393 INFO:teuthology.orchestra.run.smithi089.stdout:node-exporter.b smithi134 *:9100 running (5m) 26s ago 5m 8435k - 1.5.0 0da6a335fe13 a8cb62eeb386 2024-07-15T06:53:12.393 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0 smithi089 running (12m) 26s ago 12m 46.4M 4060M 17.2.7-1102-g9a202024 280b03ed6869 3fd6dc329fe6 2024-07-15T06:53:12.393 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1 smithi089 running (11m) 26s ago 11m 46.2M 4060M 17.2.7-1102-g9a202024 280b03ed6869 b4ff988d2ed5 2024-07-15T06:53:12.393 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2 smithi089 running (10m) 26s ago 11m 45.6M 4060M 17.2.7-1102-g9a202024 280b03ed6869 428fbfd265d1 2024-07-15T06:53:12.393 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3 smithi089 running (10m) 26s ago 10m 42.5M 4060M 17.2.7-1102-g9a202024 280b03ed6869 4f43743cd007 2024-07-15T06:53:12.393 INFO:teuthology.orchestra.run.smithi089.stdout:osd.4 smithi134 running (9m) 26s ago 9m 47.1M 4060M 17.2.7-1102-g9a202024 280b03ed6869 2ae2fa51d983 2024-07-15T06:53:12.393 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5 smithi134 running (8m) 26s ago 8m 47.0M 4060M 17.2.7-1102-g9a202024 280b03ed6869 61be14c58fc3 2024-07-15T06:53:12.393 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6 smithi134 running (7m) 26s ago 7m 49.9M 4060M 17.2.7-1102-g9a202024 280b03ed6869 f92c7dd84666 2024-07-15T06:53:12.394 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7 smithi134 running (6m) 26s ago 6m 44.2M 4060M 17.2.7-1102-g9a202024 280b03ed6869 e44d11319ff6 2024-07-15T06:53:12.394 INFO:teuthology.orchestra.run.smithi089.stdout:prometheus.a smithi134 *:9095 running (4m) 26s ago 5m 47.0M - 2.43.0 a07b618ecd1d e5f71a09faaa 2024-07-15T06:53:12.826 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:53:12 smithi089 bash[22045]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:53:12] "GET /metrics HTTP/1.1" 200 239985 "" "Prometheus/2.43.0" 2024-07-15T06:53:13.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:53:12 smithi089 bash[21793]: cluster 2024-07-15T06:53:12.214325+0000 mgr.y (mgr.24272) 208 : cluster [DBG] pgmap v184: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:53:13.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:53:12 smithi089 bash[21793]: audit 2024-07-15T06:53:12.358142+0000 mgr.y (mgr.24272) 209 : audit [DBG] from='client.24662 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T06:53:13.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:53:12 smithi089 bash[31372]: cluster 2024-07-15T06:53:12.214325+0000 mgr.y (mgr.24272) 208 : cluster [DBG] pgmap v184: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:53:13.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:53:12 smithi089 bash[31372]: audit 2024-07-15T06:53:12.358142+0000 mgr.y (mgr.24272) 209 : audit [DBG] from='client.24662 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T06:53:13.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:53:12 smithi134 bash[24068]: cluster 2024-07-15T06:53:12.214325+0000 mgr.y (mgr.24272) 208 : cluster [DBG] pgmap v184: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:53:13.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:53:12 smithi134 bash[24068]: audit 2024-07-15T06:53:12.358142+0000 mgr.y (mgr.24272) 209 : audit [DBG] from='client.24662 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T06:53:13.384 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -e sha1=6ea7aaa9f93d34dc4d21862afe00140dfddbd706 -- bash -c 'echo wait for minority of mons to upgrade' 2024-07-15T06:53:14.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:53:14 smithi134 bash[24068]: audit 2024-07-15T06:53:14.195974+0000 mon.a (mon.0) 923 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:53:14.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:53:14 smithi089 bash[21793]: audit 2024-07-15T06:53:14.195974+0000 mon.a (mon.0) 923 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:53:14.619 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:53:14 smithi089 bash[31372]: audit 2024-07-15T06:53:14.195974+0000 mon.a (mon.0) 923 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:53:15.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:53:15 smithi134 bash[24068]: cluster 2024-07-15T06:53:14.216389+0000 mgr.y (mgr.24272) 210 : cluster [DBG] pgmap v185: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:53:15.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:53:15 smithi089 bash[21793]: cluster 2024-07-15T06:53:14.216389+0000 mgr.y (mgr.24272) 210 : cluster [DBG] pgmap v185: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:53:15.619 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:53:15 smithi089 bash[31372]: cluster 2024-07-15T06:53:14.216389+0000 mgr.y (mgr.24272) 210 : cluster [DBG] pgmap v185: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:53:17.271 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:53:16 smithi134 bash[25019]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:53:16] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-07-15T06:53:17.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:53:17 smithi134 bash[24068]: cluster 2024-07-15T06:53:16.217260+0000 mgr.y (mgr.24272) 211 : cluster [DBG] pgmap v186: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:53:17.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:53:17 smithi089 bash[21793]: cluster 2024-07-15T06:53:16.217260+0000 mgr.y (mgr.24272) 211 : cluster [DBG] pgmap v186: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:53:17.619 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:53:17 smithi089 bash[31372]: cluster 2024-07-15T06:53:16.217260+0000 mgr.y (mgr.24272) 211 : cluster [DBG] pgmap v186: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:53:18.175 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T06:53:19.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:53:19 smithi134 bash[24068]: cluster 2024-07-15T06:53:18.217677+0000 mgr.y (mgr.24272) 212 : cluster [DBG] pgmap v187: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:53:19.636 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:53:19 smithi089 bash[21793]: cluster 2024-07-15T06:53:18.217677+0000 mgr.y (mgr.24272) 212 : cluster [DBG] pgmap v187: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:53:19.636 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:53:19 smithi089 bash[31372]: cluster 2024-07-15T06:53:18.217677+0000 mgr.y (mgr.24272) 212 : cluster [DBG] pgmap v187: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:53:21.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:53:21 smithi134 bash[24068]: cluster 2024-07-15T06:53:20.218693+0000 mgr.y (mgr.24272) 213 : cluster [DBG] pgmap v188: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:53:21.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:53:21 smithi089 bash[21793]: cluster 2024-07-15T06:53:20.218693+0000 mgr.y (mgr.24272) 213 : cluster [DBG] pgmap v188: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:53:21.619 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:53:21 smithi089 bash[31372]: cluster 2024-07-15T06:53:20.218693+0000 mgr.y (mgr.24272) 213 : cluster [DBG] pgmap v188: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:53:22.825 INFO:teuthology.orchestra.run.smithi089.stdout:wait for minority of mons to upgrade 2024-07-15T06:53:22.884 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:53:22 smithi089 bash[22045]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:53:22] "GET /metrics HTTP/1.1" 200 239975 "" "Prometheus/2.43.0" 2024-07-15T06:53:23.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:53:23 smithi134 bash[24068]: cluster 2024-07-15T06:53:22.219489+0000 mgr.y (mgr.24272) 214 : cluster [DBG] pgmap v189: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:53:23.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:53:23 smithi089 bash[21793]: cluster 2024-07-15T06:53:22.219489+0000 mgr.y (mgr.24272) 214 : cluster [DBG] pgmap v189: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:53:23.619 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:53:23 smithi089 bash[31372]: cluster 2024-07-15T06:53:22.219489+0000 mgr.y (mgr.24272) 214 : cluster [DBG] pgmap v189: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:53:23.793 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -e sha1=6ea7aaa9f93d34dc4d21862afe00140dfddbd706 -- bash -c 'while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done' 2024-07-15T06:53:25.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:53:25 smithi134 bash[24068]: cluster 2024-07-15T06:53:24.220055+0000 mgr.y (mgr.24272) 215 : cluster [DBG] pgmap v190: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:53:25.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:53:25 smithi089 bash[21793]: cluster 2024-07-15T06:53:24.220055+0000 mgr.y (mgr.24272) 215 : cluster [DBG] pgmap v190: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:53:25.619 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:53:25 smithi089 bash[31372]: cluster 2024-07-15T06:53:24.220055+0000 mgr.y (mgr.24272) 215 : cluster [DBG] pgmap v190: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:53:27.290 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:53:26 smithi134 bash[25019]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:53:26] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-07-15T06:53:27.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:53:27 smithi134 bash[24068]: cluster 2024-07-15T06:53:26.221270+0000 mgr.y (mgr.24272) 216 : cluster [DBG] pgmap v191: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:53:27.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:53:27 smithi089 bash[21793]: cluster 2024-07-15T06:53:26.221270+0000 mgr.y (mgr.24272) 216 : cluster [DBG] pgmap v191: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:53:27.619 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:53:27 smithi089 bash[31372]: cluster 2024-07-15T06:53:26.221270+0000 mgr.y (mgr.24272) 216 : cluster [DBG] pgmap v191: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:53:28.560 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T06:53:29.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:53:29 smithi134 bash[24068]: cluster 2024-07-15T06:53:28.221810+0000 mgr.y (mgr.24272) 217 : cluster [DBG] pgmap v192: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:53:29.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:53:29 smithi134 bash[24068]: audit 2024-07-15T06:53:29.196579+0000 mon.a (mon.0) 924 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:53:29.619 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:53:29 smithi089 bash[31372]: cluster 2024-07-15T06:53:28.221810+0000 mgr.y (mgr.24272) 217 : cluster [DBG] pgmap v192: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:53:29.619 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:53:29 smithi089 bash[31372]: audit 2024-07-15T06:53:29.196579+0000 mon.a (mon.0) 924 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:53:29.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:53:29 smithi089 bash[21793]: cluster 2024-07-15T06:53:28.221810+0000 mgr.y (mgr.24272) 217 : cluster [DBG] pgmap v192: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:53:29.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:53:29 smithi089 bash[21793]: audit 2024-07-15T06:53:29.196579+0000 mon.a (mon.0) 924 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:53:31.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:53:31 smithi134 bash[24068]: cluster 2024-07-15T06:53:30.223048+0000 mgr.y (mgr.24272) 218 : cluster [DBG] pgmap v193: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:53:31.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:53:31 smithi089 bash[21793]: cluster 2024-07-15T06:53:30.223048+0000 mgr.y (mgr.24272) 218 : cluster [DBG] pgmap v193: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:53:31.619 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:53:31 smithi089 bash[31372]: cluster 2024-07-15T06:53:30.223048+0000 mgr.y (mgr.24272) 218 : cluster [DBG] pgmap v193: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:53:33.119 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:53:32 smithi089 bash[22045]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:53:32] "GET /metrics HTTP/1.1" 200 239937 "" "Prometheus/2.43.0" 2024-07-15T06:53:33.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:53:33 smithi134 bash[24068]: cluster 2024-07-15T06:53:32.223721+0000 mgr.y (mgr.24272) 219 : cluster [DBG] pgmap v194: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:53:33.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:53:33 smithi089 bash[21793]: cluster 2024-07-15T06:53:32.223721+0000 mgr.y (mgr.24272) 219 : cluster [DBG] pgmap v194: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:53:33.619 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:53:33 smithi089 bash[31372]: cluster 2024-07-15T06:53:32.223721+0000 mgr.y (mgr.24272) 219 : cluster [DBG] pgmap v194: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:53:35.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:53:35 smithi089 bash[21793]: cluster 2024-07-15T06:53:34.224324+0000 mgr.y (mgr.24272) 220 : cluster [DBG] pgmap v195: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:53:35.619 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:53:35 smithi089 bash[31372]: cluster 2024-07-15T06:53:34.224324+0000 mgr.y (mgr.24272) 220 : cluster [DBG] pgmap v195: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:53:35.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:53:35 smithi134 bash[24068]: cluster 2024-07-15T06:53:34.224324+0000 mgr.y (mgr.24272) 220 : cluster [DBG] pgmap v195: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:53:37.304 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:53:36 smithi134 bash[25019]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:53:36] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-07-15T06:53:37.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:53:37 smithi089 bash[21793]: cluster 2024-07-15T06:53:36.225448+0000 mgr.y (mgr.24272) 221 : cluster [DBG] pgmap v196: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:53:37.619 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:53:37 smithi089 bash[31372]: cluster 2024-07-15T06:53:36.225448+0000 mgr.y (mgr.24272) 221 : cluster [DBG] pgmap v196: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:53:37.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:53:37 smithi134 bash[24068]: cluster 2024-07-15T06:53:36.225448+0000 mgr.y (mgr.24272) 221 : cluster [DBG] pgmap v196: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:53:39.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:53:39 smithi089 bash[21793]: cluster 2024-07-15T06:53:38.226037+0000 mgr.y (mgr.24272) 222 : cluster [DBG] pgmap v197: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:53:39.619 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:53:39 smithi089 bash[31372]: cluster 2024-07-15T06:53:38.226037+0000 mgr.y (mgr.24272) 222 : cluster [DBG] pgmap v197: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:53:39.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:53:39 smithi134 bash[24068]: cluster 2024-07-15T06:53:38.226037+0000 mgr.y (mgr.24272) 222 : cluster [DBG] pgmap v197: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:53:41.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:53:41 smithi089 bash[21793]: cluster 2024-07-15T06:53:40.227072+0000 mgr.y (mgr.24272) 223 : cluster [DBG] pgmap v198: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:53:41.619 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:53:41 smithi089 bash[31372]: cluster 2024-07-15T06:53:40.227072+0000 mgr.y (mgr.24272) 223 : cluster [DBG] pgmap v198: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:53:41.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:53:41 smithi134 bash[24068]: cluster 2024-07-15T06:53:40.227072+0000 mgr.y (mgr.24272) 223 : cluster [DBG] pgmap v198: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:53:43.042 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:53:42 smithi089 bash[22045]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:53:42] "GET /metrics HTTP/1.1" 200 239937 "" "Prometheus/2.43.0" 2024-07-15T06:53:43.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:53:43 smithi089 bash[21793]: cluster 2024-07-15T06:53:42.227684+0000 mgr.y (mgr.24272) 224 : cluster [DBG] pgmap v199: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:53:43.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:53:43 smithi089 bash[21793]: audit 2024-07-15T06:53:43.110074+0000 mon.c (mon.2) 30 : audit [DBG] from='client.? 172.21.15.89:0/1643664102' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:53:43.619 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:53:43 smithi089 bash[31372]: cluster 2024-07-15T06:53:42.227684+0000 mgr.y (mgr.24272) 224 : cluster [DBG] pgmap v199: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:53:43.619 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:53:43 smithi089 bash[31372]: audit 2024-07-15T06:53:43.110074+0000 mon.c (mon.2) 30 : audit [DBG] from='client.? 172.21.15.89:0/1643664102' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:53:43.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:53:43 smithi134 bash[24068]: cluster 2024-07-15T06:53:42.227684+0000 mgr.y (mgr.24272) 224 : cluster [DBG] pgmap v199: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:53:43.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:53:43 smithi134 bash[24068]: audit 2024-07-15T06:53:43.110074+0000 mon.c (mon.2) 30 : audit [DBG] from='client.? 172.21.15.89:0/1643664102' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:53:45.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:53:44 smithi134 bash[24068]: audit 2024-07-15T06:53:43.643056+0000 mon.a (mon.0) 925 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:53:45.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:53:44 smithi134 bash[24068]: cephadm 2024-07-15T06:53:43.643912+0000 mgr.y (mgr.24272) 225 : cephadm [INF] Upgrade: Target is version 18.2.2-1609-g6ea7aaa9 (unknown) 2024-07-15T06:53:45.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:53:44 smithi134 bash[24068]: cephadm 2024-07-15T06:53:43.644076+0000 mgr.y (mgr.24272) 226 : cephadm [INF] Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:badc66b92d2a2cfbbce7f293b5accdc50f486d2b2d966876e46fe28d05f464f1, digests ['quay.ceph.io/ceph-ci/ceph@sha256:badc66b92d2a2cfbbce7f293b5accdc50f486d2b2d966876e46fe28d05f464f1'] 2024-07-15T06:53:45.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:53:44 smithi134 bash[24068]: audit 2024-07-15T06:53:43.646195+0000 mon.a (mon.0) 926 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:53:45.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:53:44 smithi134 bash[24068]: audit 2024-07-15T06:53:43.656368+0000 mon.a (mon.0) 927 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:53:45.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:53:44 smithi134 bash[24068]: cephadm 2024-07-15T06:53:43.658184+0000 mgr.y (mgr.24272) 227 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.y) 2024-07-15T06:53:45.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:53:44 smithi134 bash[24068]: audit 2024-07-15T06:53:44.196370+0000 mon.a (mon.0) 928 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:53:45.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:53:44 smithi089 bash[31372]: audit 2024-07-15T06:53:43.643056+0000 mon.a (mon.0) 925 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:53:45.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:53:44 smithi089 bash[31372]: cephadm 2024-07-15T06:53:43.643912+0000 mgr.y (mgr.24272) 225 : cephadm [INF] Upgrade: Target is version 18.2.2-1609-g6ea7aaa9 (unknown) 2024-07-15T06:53:45.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:53:44 smithi089 bash[31372]: cephadm 2024-07-15T06:53:43.644076+0000 mgr.y (mgr.24272) 226 : cephadm [INF] Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:badc66b92d2a2cfbbce7f293b5accdc50f486d2b2d966876e46fe28d05f464f1, digests ['quay.ceph.io/ceph-ci/ceph@sha256:badc66b92d2a2cfbbce7f293b5accdc50f486d2b2d966876e46fe28d05f464f1'] 2024-07-15T06:53:45.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:53:44 smithi089 bash[31372]: audit 2024-07-15T06:53:43.646195+0000 mon.a (mon.0) 926 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:53:45.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:53:44 smithi089 bash[31372]: audit 2024-07-15T06:53:43.656368+0000 mon.a (mon.0) 927 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:53:45.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:53:44 smithi089 bash[31372]: cephadm 2024-07-15T06:53:43.658184+0000 mgr.y (mgr.24272) 227 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.y) 2024-07-15T06:53:45.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:53:44 smithi089 bash[31372]: audit 2024-07-15T06:53:44.196370+0000 mon.a (mon.0) 928 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:53:45.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:53:44 smithi089 bash[21793]: audit 2024-07-15T06:53:43.643056+0000 mon.a (mon.0) 925 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:53:45.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:53:44 smithi089 bash[21793]: cephadm 2024-07-15T06:53:43.643912+0000 mgr.y (mgr.24272) 225 : cephadm [INF] Upgrade: Target is version 18.2.2-1609-g6ea7aaa9 (unknown) 2024-07-15T06:53:45.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:53:44 smithi089 bash[21793]: cephadm 2024-07-15T06:53:43.644076+0000 mgr.y (mgr.24272) 226 : cephadm [INF] Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:badc66b92d2a2cfbbce7f293b5accdc50f486d2b2d966876e46fe28d05f464f1, digests ['quay.ceph.io/ceph-ci/ceph@sha256:badc66b92d2a2cfbbce7f293b5accdc50f486d2b2d966876e46fe28d05f464f1'] 2024-07-15T06:53:45.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:53:44 smithi089 bash[21793]: audit 2024-07-15T06:53:43.646195+0000 mon.a (mon.0) 926 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:53:45.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:53:44 smithi089 bash[21793]: audit 2024-07-15T06:53:43.656368+0000 mon.a (mon.0) 927 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:53:45.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:53:44 smithi089 bash[21793]: cephadm 2024-07-15T06:53:43.658184+0000 mgr.y (mgr.24272) 227 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.y) 2024-07-15T06:53:45.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:53:44 smithi089 bash[21793]: audit 2024-07-15T06:53:44.196370+0000 mon.a (mon.0) 928 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:53:46.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:53:45 smithi134 bash[24068]: cephadm 2024-07-15T06:53:43.909487+0000 mgr.y (mgr.24272) 228 : cephadm [INF] Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:badc66b92d2a2cfbbce7f293b5accdc50f486d2b2d966876e46fe28d05f464f1 on smithi134 2024-07-15T06:53:46.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:53:45 smithi134 bash[24068]: cluster 2024-07-15T06:53:44.228277+0000 mgr.y (mgr.24272) 229 : cluster [DBG] pgmap v200: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:53:46.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:53:45 smithi134 bash[24068]: audit 2024-07-15T06:53:45.474861+0000 mon.a (mon.0) 929 : audit [DBG] from='client.? 172.21.15.89:0/3850103422' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:53:46.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:53:45 smithi089 bash[21793]: cephadm 2024-07-15T06:53:43.909487+0000 mgr.y (mgr.24272) 228 : cephadm [INF] Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:badc66b92d2a2cfbbce7f293b5accdc50f486d2b2d966876e46fe28d05f464f1 on smithi134 2024-07-15T06:53:46.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:53:45 smithi089 bash[21793]: cluster 2024-07-15T06:53:44.228277+0000 mgr.y (mgr.24272) 229 : cluster [DBG] pgmap v200: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:53:46.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:53:45 smithi089 bash[21793]: audit 2024-07-15T06:53:45.474861+0000 mon.a (mon.0) 929 : audit [DBG] from='client.? 172.21.15.89:0/3850103422' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:53:46.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:53:45 smithi089 bash[31372]: cephadm 2024-07-15T06:53:43.909487+0000 mgr.y (mgr.24272) 228 : cephadm [INF] Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:badc66b92d2a2cfbbce7f293b5accdc50f486d2b2d966876e46fe28d05f464f1 on smithi134 2024-07-15T06:53:46.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:53:45 smithi089 bash[31372]: cluster 2024-07-15T06:53:44.228277+0000 mgr.y (mgr.24272) 229 : cluster [DBG] pgmap v200: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:53:46.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:53:45 smithi089 bash[31372]: audit 2024-07-15T06:53:45.474861+0000 mon.a (mon.0) 929 : audit [DBG] from='client.? 172.21.15.89:0/3850103422' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:53:47.304 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:53:46 smithi134 bash[25019]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:53:46] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-07-15T06:53:48.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:53:47 smithi134 bash[24068]: cluster 2024-07-15T06:53:46.229382+0000 mgr.y (mgr.24272) 230 : cluster [DBG] pgmap v201: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:53:48.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:53:47 smithi089 bash[21793]: cluster 2024-07-15T06:53:46.229382+0000 mgr.y (mgr.24272) 230 : cluster [DBG] pgmap v201: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:53:48.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:53:47 smithi089 bash[31372]: cluster 2024-07-15T06:53:46.229382+0000 mgr.y (mgr.24272) 230 : cluster [DBG] pgmap v201: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:53:49.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:53:48 smithi134 bash[24068]: audit 2024-07-15T06:53:47.841444+0000 mon.a (mon.0) 930 : audit [DBG] from='client.? 172.21.15.89:0/3062544247' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:53:49.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:53:48 smithi089 bash[21793]: audit 2024-07-15T06:53:47.841444+0000 mon.a (mon.0) 930 : audit [DBG] from='client.? 172.21.15.89:0/3062544247' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:53:49.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:53:48 smithi089 bash[31372]: audit 2024-07-15T06:53:47.841444+0000 mon.a (mon.0) 930 : audit [DBG] from='client.? 172.21.15.89:0/3062544247' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:53:50.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:53:49 smithi134 bash[24068]: cluster 2024-07-15T06:53:48.229988+0000 mgr.y (mgr.24272) 231 : cluster [DBG] pgmap v202: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:53:50.118 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:53:49 smithi089 bash[21793]: cluster 2024-07-15T06:53:48.229988+0000 mgr.y (mgr.24272) 231 : cluster [DBG] pgmap v202: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:53:50.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:53:49 smithi089 bash[31372]: cluster 2024-07-15T06:53:48.229988+0000 mgr.y (mgr.24272) 231 : cluster [DBG] pgmap v202: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:53:51.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:53:50 smithi134 bash[24068]: audit 2024-07-15T06:53:50.202568+0000 mon.c (mon.2) 31 : audit [DBG] from='client.? 172.21.15.89:0/2296587458' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:53:51.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:53:50 smithi089 bash[21793]: audit 2024-07-15T06:53:50.202568+0000 mon.c (mon.2) 31 : audit [DBG] from='client.? 172.21.15.89:0/2296587458' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:53:51.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:53:50 smithi089 bash[31372]: audit 2024-07-15T06:53:50.202568+0000 mon.c (mon.2) 31 : audit [DBG] from='client.? 172.21.15.89:0/2296587458' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:53:52.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:53:51 smithi134 bash[24068]: cluster 2024-07-15T06:53:50.231025+0000 mgr.y (mgr.24272) 232 : cluster [DBG] pgmap v203: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:53:52.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:53:51 smithi089 bash[21793]: cluster 2024-07-15T06:53:50.231025+0000 mgr.y (mgr.24272) 232 : cluster [DBG] pgmap v203: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:53:52.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:53:51 smithi089 bash[31372]: cluster 2024-07-15T06:53:50.231025+0000 mgr.y (mgr.24272) 232 : cluster [DBG] pgmap v203: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:53:53.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:53:52 smithi134 bash[24068]: audit 2024-07-15T06:53:52.543619+0000 mon.a (mon.0) 931 : audit [DBG] from='client.? 172.21.15.89:0/3621564354' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:53:53.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:53:52 smithi089 bash[21793]: audit 2024-07-15T06:53:52.543619+0000 mon.a (mon.0) 931 : audit [DBG] from='client.? 172.21.15.89:0/3621564354' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:53:53.119 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:53:52 smithi089 bash[22045]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:53:52] "GET /metrics HTTP/1.1" 200 239936 "" "Prometheus/2.43.0" 2024-07-15T06:53:53.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:53:52 smithi089 bash[31372]: audit 2024-07-15T06:53:52.543619+0000 mon.a (mon.0) 931 : audit [DBG] from='client.? 172.21.15.89:0/3621564354' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:53:54.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:53:53 smithi134 bash[24068]: cluster 2024-07-15T06:53:52.231738+0000 mgr.y (mgr.24272) 233 : cluster [DBG] pgmap v204: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:53:54.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:53:53 smithi089 bash[21793]: cluster 2024-07-15T06:53:52.231738+0000 mgr.y (mgr.24272) 233 : cluster [DBG] pgmap v204: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:53:54.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:53:53 smithi089 bash[31372]: cluster 2024-07-15T06:53:52.231738+0000 mgr.y (mgr.24272) 233 : cluster [DBG] pgmap v204: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:53:56.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:53:55 smithi134 bash[24068]: cluster 2024-07-15T06:53:54.232373+0000 mgr.y (mgr.24272) 234 : cluster [DBG] pgmap v205: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:53:56.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:53:55 smithi134 bash[24068]: audit 2024-07-15T06:53:54.907331+0000 mon.a (mon.0) 932 : audit [DBG] from='client.? 172.21.15.89:0/309176504' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:53:56.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:53:55 smithi089 bash[31372]: cluster 2024-07-15T06:53:54.232373+0000 mgr.y (mgr.24272) 234 : cluster [DBG] pgmap v205: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:53:56.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:53:55 smithi089 bash[31372]: audit 2024-07-15T06:53:54.907331+0000 mon.a (mon.0) 932 : audit [DBG] from='client.? 172.21.15.89:0/309176504' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:53:56.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:53:55 smithi089 bash[21793]: cluster 2024-07-15T06:53:54.232373+0000 mgr.y (mgr.24272) 234 : cluster [DBG] pgmap v205: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:53:56.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:53:55 smithi089 bash[21793]: audit 2024-07-15T06:53:54.907331+0000 mon.a (mon.0) 932 : audit [DBG] from='client.? 172.21.15.89:0/309176504' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:53:57.304 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:53:56 smithi134 bash[25019]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:53:56] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-07-15T06:53:58.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:53:57 smithi134 bash[24068]: cluster 2024-07-15T06:53:56.233478+0000 mgr.y (mgr.24272) 235 : cluster [DBG] pgmap v206: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:53:58.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:53:57 smithi134 bash[24068]: audit 2024-07-15T06:53:57.249911+0000 mon.c (mon.2) 32 : audit [DBG] from='client.? 172.21.15.89:0/3160127953' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:53:58.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:53:57 smithi089 bash[31372]: cluster 2024-07-15T06:53:56.233478+0000 mgr.y (mgr.24272) 235 : cluster [DBG] pgmap v206: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:53:58.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:53:57 smithi089 bash[31372]: audit 2024-07-15T06:53:57.249911+0000 mon.c (mon.2) 32 : audit [DBG] from='client.? 172.21.15.89:0/3160127953' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:53:58.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:53:57 smithi089 bash[21793]: cluster 2024-07-15T06:53:56.233478+0000 mgr.y (mgr.24272) 235 : cluster [DBG] pgmap v206: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:53:58.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:53:57 smithi089 bash[21793]: audit 2024-07-15T06:53:57.249911+0000 mon.c (mon.2) 32 : audit [DBG] from='client.? 172.21.15.89:0/3160127953' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:54:00.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:53:59 smithi134 bash[24068]: cluster 2024-07-15T06:53:58.234086+0000 mgr.y (mgr.24272) 236 : cluster [DBG] pgmap v207: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:54:00.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:53:59 smithi134 bash[24068]: audit 2024-07-15T06:53:59.196813+0000 mon.a (mon.0) 933 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:54:00.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:53:59 smithi134 bash[24068]: audit 2024-07-15T06:53:59.610962+0000 mon.c (mon.2) 33 : audit [DBG] from='client.? 172.21.15.89:0/2336657447' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:54:00.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:53:59 smithi089 bash[21793]: cluster 2024-07-15T06:53:58.234086+0000 mgr.y (mgr.24272) 236 : cluster [DBG] pgmap v207: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:54:00.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:53:59 smithi089 bash[21793]: audit 2024-07-15T06:53:59.196813+0000 mon.a (mon.0) 933 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:54:00.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:53:59 smithi089 bash[21793]: audit 2024-07-15T06:53:59.610962+0000 mon.c (mon.2) 33 : audit [DBG] from='client.? 172.21.15.89:0/2336657447' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:54:00.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:53:59 smithi089 bash[31372]: cluster 2024-07-15T06:53:58.234086+0000 mgr.y (mgr.24272) 236 : cluster [DBG] pgmap v207: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:54:00.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:53:59 smithi089 bash[31372]: audit 2024-07-15T06:53:59.196813+0000 mon.a (mon.0) 933 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:54:00.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:53:59 smithi089 bash[31372]: audit 2024-07-15T06:53:59.610962+0000 mon.c (mon.2) 33 : audit [DBG] from='client.? 172.21.15.89:0/2336657447' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:54:02.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:54:01 smithi134 bash[24068]: cluster 2024-07-15T06:54:00.235117+0000 mgr.y (mgr.24272) 237 : cluster [DBG] pgmap v208: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:54:02.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:54:01 smithi089 bash[21793]: cluster 2024-07-15T06:54:00.235117+0000 mgr.y (mgr.24272) 237 : cluster [DBG] pgmap v208: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:54:02.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:54:01 smithi089 bash[31372]: cluster 2024-07-15T06:54:00.235117+0000 mgr.y (mgr.24272) 237 : cluster [DBG] pgmap v208: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:54:03.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:54:02 smithi134 bash[24068]: audit 2024-07-15T06:54:01.966239+0000 mon.c (mon.2) 34 : audit [DBG] from='client.? 172.21.15.89:0/1871800935' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:54:03.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:54:02 smithi089 bash[21793]: audit 2024-07-15T06:54:01.966239+0000 mon.c (mon.2) 34 : audit [DBG] from='client.? 172.21.15.89:0/1871800935' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:54:03.119 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:54:02 smithi089 bash[22045]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:54:02] "GET /metrics HTTP/1.1" 200 239944 "" "Prometheus/2.43.0" 2024-07-15T06:54:03.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:54:02 smithi089 bash[31372]: audit 2024-07-15T06:54:01.966239+0000 mon.c (mon.2) 34 : audit [DBG] from='client.? 172.21.15.89:0/1871800935' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:54:04.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:54:03 smithi134 bash[24068]: cluster 2024-07-15T06:54:02.235719+0000 mgr.y (mgr.24272) 238 : cluster [DBG] pgmap v209: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:54:04.118 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:54:03 smithi089 bash[21793]: cluster 2024-07-15T06:54:02.235719+0000 mgr.y (mgr.24272) 238 : cluster [DBG] pgmap v209: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:54:04.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:54:03 smithi089 bash[31372]: cluster 2024-07-15T06:54:02.235719+0000 mgr.y (mgr.24272) 238 : cluster [DBG] pgmap v209: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:54:05.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:54:04 smithi134 bash[24068]: audit 2024-07-15T06:54:04.310042+0000 mon.a (mon.0) 934 : audit [DBG] from='client.? 172.21.15.89:0/1937232589' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:54:05.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:54:04 smithi089 bash[21793]: audit 2024-07-15T06:54:04.310042+0000 mon.a (mon.0) 934 : audit [DBG] from='client.? 172.21.15.89:0/1937232589' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:54:05.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:54:04 smithi089 bash[31372]: audit 2024-07-15T06:54:04.310042+0000 mon.a (mon.0) 934 : audit [DBG] from='client.? 172.21.15.89:0/1937232589' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:54:06.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:54:05 smithi134 bash[24068]: cluster 2024-07-15T06:54:04.236311+0000 mgr.y (mgr.24272) 239 : cluster [DBG] pgmap v210: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:54:06.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:54:05 smithi089 bash[31372]: cluster 2024-07-15T06:54:04.236311+0000 mgr.y (mgr.24272) 239 : cluster [DBG] pgmap v210: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:54:06.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:54:05 smithi089 bash[21793]: cluster 2024-07-15T06:54:04.236311+0000 mgr.y (mgr.24272) 239 : cluster [DBG] pgmap v210: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:54:07.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:54:06 smithi134 bash[24068]: audit 2024-07-15T06:54:06.674337+0000 mon.c (mon.2) 35 : audit [DBG] from='client.? 172.21.15.89:0/2518593822' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:54:07.054 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:54:06 smithi134 bash[25019]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:54:06] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-07-15T06:54:07.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:54:06 smithi089 bash[31372]: audit 2024-07-15T06:54:06.674337+0000 mon.c (mon.2) 35 : audit [DBG] from='client.? 172.21.15.89:0/2518593822' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:54:07.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:54:06 smithi089 bash[21793]: audit 2024-07-15T06:54:06.674337+0000 mon.c (mon.2) 35 : audit [DBG] from='client.? 172.21.15.89:0/2518593822' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:54:08.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:54:07 smithi134 bash[24068]: cluster 2024-07-15T06:54:06.237417+0000 mgr.y (mgr.24272) 240 : cluster [DBG] pgmap v211: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:54:08.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:54:07 smithi089 bash[21793]: cluster 2024-07-15T06:54:06.237417+0000 mgr.y (mgr.24272) 240 : cluster [DBG] pgmap v211: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:54:08.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:54:07 smithi089 bash[31372]: cluster 2024-07-15T06:54:06.237417+0000 mgr.y (mgr.24272) 240 : cluster [DBG] pgmap v211: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:54:10.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:54:09 smithi134 bash[24068]: cluster 2024-07-15T06:54:08.238058+0000 mgr.y (mgr.24272) 241 : cluster [DBG] pgmap v212: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:54:10.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:54:09 smithi134 bash[24068]: audit 2024-07-15T06:54:09.036461+0000 mon.a (mon.0) 935 : audit [DBG] from='client.? 172.21.15.89:0/51557932' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:54:10.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:54:09 smithi089 bash[21793]: cluster 2024-07-15T06:54:08.238058+0000 mgr.y (mgr.24272) 241 : cluster [DBG] pgmap v212: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:54:10.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:54:09 smithi089 bash[21793]: audit 2024-07-15T06:54:09.036461+0000 mon.a (mon.0) 935 : audit [DBG] from='client.? 172.21.15.89:0/51557932' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:54:10.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:54:09 smithi089 bash[31372]: cluster 2024-07-15T06:54:08.238058+0000 mgr.y (mgr.24272) 241 : cluster [DBG] pgmap v212: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:54:10.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:54:09 smithi089 bash[31372]: audit 2024-07-15T06:54:09.036461+0000 mon.a (mon.0) 935 : audit [DBG] from='client.? 172.21.15.89:0/51557932' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:54:12.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:54:11 smithi134 bash[24068]: cluster 2024-07-15T06:54:10.239114+0000 mgr.y (mgr.24272) 242 : cluster [DBG] pgmap v213: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:54:12.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:54:11 smithi134 bash[24068]: audit 2024-07-15T06:54:11.387635+0000 mon.c (mon.2) 36 : audit [DBG] from='client.? 172.21.15.89:0/1688041094' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:54:12.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:54:11 smithi089 bash[21793]: cluster 2024-07-15T06:54:10.239114+0000 mgr.y (mgr.24272) 242 : cluster [DBG] pgmap v213: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:54:12.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:54:11 smithi089 bash[21793]: audit 2024-07-15T06:54:11.387635+0000 mon.c (mon.2) 36 : audit [DBG] from='client.? 172.21.15.89:0/1688041094' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:54:12.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:54:11 smithi089 bash[31372]: cluster 2024-07-15T06:54:10.239114+0000 mgr.y (mgr.24272) 242 : cluster [DBG] pgmap v213: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:54:12.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:54:11 smithi089 bash[31372]: audit 2024-07-15T06:54:11.387635+0000 mon.c (mon.2) 36 : audit [DBG] from='client.? 172.21.15.89:0/1688041094' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:54:13.119 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:54:12 smithi089 bash[22045]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:54:12] "GET /metrics HTTP/1.1" 200 239944 "" "Prometheus/2.43.0" 2024-07-15T06:54:14.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:54:13 smithi134 bash[24068]: cluster 2024-07-15T06:54:12.239766+0000 mgr.y (mgr.24272) 243 : cluster [DBG] pgmap v214: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:54:14.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:54:13 smithi089 bash[21793]: cluster 2024-07-15T06:54:12.239766+0000 mgr.y (mgr.24272) 243 : cluster [DBG] pgmap v214: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:54:14.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:54:13 smithi089 bash[31372]: cluster 2024-07-15T06:54:12.239766+0000 mgr.y (mgr.24272) 243 : cluster [DBG] pgmap v214: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:54:15.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:54:14 smithi134 bash[24068]: audit 2024-07-15T06:54:13.751455+0000 mon.c (mon.2) 37 : audit [DBG] from='client.? 172.21.15.89:0/2916102387' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:54:15.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:54:14 smithi134 bash[24068]: audit 2024-07-15T06:54:14.197369+0000 mon.a (mon.0) 936 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:54:15.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:54:14 smithi089 bash[21793]: audit 2024-07-15T06:54:13.751455+0000 mon.c (mon.2) 37 : audit [DBG] from='client.? 172.21.15.89:0/2916102387' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:54:15.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:54:14 smithi089 bash[21793]: audit 2024-07-15T06:54:14.197369+0000 mon.a (mon.0) 936 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:54:15.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:54:14 smithi089 bash[31372]: audit 2024-07-15T06:54:13.751455+0000 mon.c (mon.2) 37 : audit [DBG] from='client.? 172.21.15.89:0/2916102387' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:54:15.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:54:14 smithi089 bash[31372]: audit 2024-07-15T06:54:14.197369+0000 mon.a (mon.0) 936 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:54:16.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:54:15 smithi134 bash[24068]: cluster 2024-07-15T06:54:14.240363+0000 mgr.y (mgr.24272) 244 : cluster [DBG] pgmap v215: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:54:16.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:54:15 smithi089 bash[21793]: cluster 2024-07-15T06:54:14.240363+0000 mgr.y (mgr.24272) 244 : cluster [DBG] pgmap v215: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:54:16.143 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:54:15 smithi089 bash[31372]: cluster 2024-07-15T06:54:14.240363+0000 mgr.y (mgr.24272) 244 : cluster [DBG] pgmap v215: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:54:17.054 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:54:16 smithi134 bash[25019]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:54:16] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-07-15T06:54:17.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:54:16 smithi134 bash[24068]: audit 2024-07-15T06:54:16.104086+0000 mon.a (mon.0) 937 : audit [DBG] from='client.? 172.21.15.89:0/665193173' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:54:17.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:54:16 smithi089 bash[21793]: audit 2024-07-15T06:54:16.104086+0000 mon.a (mon.0) 937 : audit [DBG] from='client.? 172.21.15.89:0/665193173' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:54:17.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:54:16 smithi089 bash[31372]: audit 2024-07-15T06:54:16.104086+0000 mon.a (mon.0) 937 : audit [DBG] from='client.? 172.21.15.89:0/665193173' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:54:18.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:54:17 smithi134 bash[24068]: cluster 2024-07-15T06:54:16.241493+0000 mgr.y (mgr.24272) 245 : cluster [DBG] pgmap v216: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:54:18.118 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:54:17 smithi089 bash[31372]: cluster 2024-07-15T06:54:16.241493+0000 mgr.y (mgr.24272) 245 : cluster [DBG] pgmap v216: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:54:18.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:54:17 smithi089 bash[21793]: cluster 2024-07-15T06:54:16.241493+0000 mgr.y (mgr.24272) 245 : cluster [DBG] pgmap v216: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:54:19.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:54:18 smithi134 bash[24068]: audit 2024-07-15T06:54:18.466912+0000 mon.a (mon.0) 938 : audit [DBG] from='client.? 172.21.15.89:0/154383009' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:54:19.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:54:18 smithi089 bash[31372]: audit 2024-07-15T06:54:18.466912+0000 mon.a (mon.0) 938 : audit [DBG] from='client.? 172.21.15.89:0/154383009' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:54:19.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:54:18 smithi089 bash[21793]: audit 2024-07-15T06:54:18.466912+0000 mon.a (mon.0) 938 : audit [DBG] from='client.? 172.21.15.89:0/154383009' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:54:20.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:54:19 smithi134 bash[24068]: cluster 2024-07-15T06:54:18.241879+0000 mgr.y (mgr.24272) 246 : cluster [DBG] pgmap v217: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:54:20.118 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:54:19 smithi089 bash[31372]: cluster 2024-07-15T06:54:18.241879+0000 mgr.y (mgr.24272) 246 : cluster [DBG] pgmap v217: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:54:20.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:54:19 smithi089 bash[21793]: cluster 2024-07-15T06:54:18.241879+0000 mgr.y (mgr.24272) 246 : cluster [DBG] pgmap v217: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:54:22.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:54:21 smithi134 bash[24068]: cluster 2024-07-15T06:54:20.242880+0000 mgr.y (mgr.24272) 247 : cluster [DBG] pgmap v218: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:54:22.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:54:21 smithi134 bash[24068]: audit 2024-07-15T06:54:20.821183+0000 mon.a (mon.0) 939 : audit [DBG] from='client.? 172.21.15.89:0/1541432209' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:54:22.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:54:21 smithi089 bash[21793]: cluster 2024-07-15T06:54:20.242880+0000 mgr.y (mgr.24272) 247 : cluster [DBG] pgmap v218: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:54:22.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:54:21 smithi089 bash[21793]: audit 2024-07-15T06:54:20.821183+0000 mon.a (mon.0) 939 : audit [DBG] from='client.? 172.21.15.89:0/1541432209' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:54:22.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:54:21 smithi089 bash[31372]: cluster 2024-07-15T06:54:20.242880+0000 mgr.y (mgr.24272) 247 : cluster [DBG] pgmap v218: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:54:22.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:54:21 smithi089 bash[31372]: audit 2024-07-15T06:54:20.821183+0000 mon.a (mon.0) 939 : audit [DBG] from='client.? 172.21.15.89:0/1541432209' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:54:23.119 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:54:22 smithi089 bash[22045]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:54:22] "GET /metrics HTTP/1.1" 200 239938 "" "Prometheus/2.43.0" 2024-07-15T06:54:24.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:54:23 smithi134 bash[24068]: cluster 2024-07-15T06:54:22.243514+0000 mgr.y (mgr.24272) 248 : cluster [DBG] pgmap v219: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:54:24.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:54:23 smithi134 bash[24068]: audit 2024-07-15T06:54:23.175340+0000 mon.a (mon.0) 940 : audit [DBG] from='client.? 172.21.15.89:0/3392333471' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:54:24.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:54:23 smithi089 bash[21793]: cluster 2024-07-15T06:54:22.243514+0000 mgr.y (mgr.24272) 248 : cluster [DBG] pgmap v219: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:54:24.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:54:23 smithi089 bash[21793]: audit 2024-07-15T06:54:23.175340+0000 mon.a (mon.0) 940 : audit [DBG] from='client.? 172.21.15.89:0/3392333471' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:54:24.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:54:23 smithi089 bash[31372]: cluster 2024-07-15T06:54:22.243514+0000 mgr.y (mgr.24272) 248 : cluster [DBG] pgmap v219: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:54:24.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:54:23 smithi089 bash[31372]: audit 2024-07-15T06:54:23.175340+0000 mon.a (mon.0) 940 : audit [DBG] from='client.? 172.21.15.89:0/3392333471' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:54:26.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:54:25 smithi134 bash[24068]: cluster 2024-07-15T06:54:24.244147+0000 mgr.y (mgr.24272) 249 : cluster [DBG] pgmap v220: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:54:26.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:54:25 smithi134 bash[24068]: audit 2024-07-15T06:54:25.535380+0000 mon.a (mon.0) 941 : audit [DBG] from='client.? 172.21.15.89:0/2836367986' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:54:26.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:54:25 smithi089 bash[21793]: cluster 2024-07-15T06:54:24.244147+0000 mgr.y (mgr.24272) 249 : cluster [DBG] pgmap v220: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:54:26.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:54:25 smithi089 bash[21793]: audit 2024-07-15T06:54:25.535380+0000 mon.a (mon.0) 941 : audit [DBG] from='client.? 172.21.15.89:0/2836367986' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:54:26.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:54:25 smithi089 bash[31372]: cluster 2024-07-15T06:54:24.244147+0000 mgr.y (mgr.24272) 249 : cluster [DBG] pgmap v220: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:54:26.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:54:25 smithi089 bash[31372]: audit 2024-07-15T06:54:25.535380+0000 mon.a (mon.0) 941 : audit [DBG] from='client.? 172.21.15.89:0/2836367986' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:54:27.304 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:54:26 smithi134 bash[25019]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:54:26] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-07-15T06:54:28.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:54:27 smithi134 bash[24068]: cluster 2024-07-15T06:54:26.245247+0000 mgr.y (mgr.24272) 250 : cluster [DBG] pgmap v221: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:54:28.118 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:54:27 smithi089 bash[21793]: cluster 2024-07-15T06:54:26.245247+0000 mgr.y (mgr.24272) 250 : cluster [DBG] pgmap v221: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:54:28.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:54:27 smithi089 bash[31372]: cluster 2024-07-15T06:54:26.245247+0000 mgr.y (mgr.24272) 250 : cluster [DBG] pgmap v221: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:54:29.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:54:28 smithi134 bash[24068]: audit 2024-07-15T06:54:27.889472+0000 mon.a (mon.0) 942 : audit [DBG] from='client.? 172.21.15.89:0/2408632299' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:54:29.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:54:28 smithi089 bash[21793]: audit 2024-07-15T06:54:27.889472+0000 mon.a (mon.0) 942 : audit [DBG] from='client.? 172.21.15.89:0/2408632299' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:54:29.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:54:28 smithi089 bash[31372]: audit 2024-07-15T06:54:27.889472+0000 mon.a (mon.0) 942 : audit [DBG] from='client.? 172.21.15.89:0/2408632299' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:54:30.118 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:54:29 smithi089 bash[21793]: cluster 2024-07-15T06:54:28.245899+0000 mgr.y (mgr.24272) 251 : cluster [DBG] pgmap v222: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:54:30.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:54:29 smithi089 bash[21793]: audit 2024-07-15T06:54:29.197615+0000 mon.a (mon.0) 943 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:54:30.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:54:29 smithi089 bash[31372]: cluster 2024-07-15T06:54:28.245899+0000 mgr.y (mgr.24272) 251 : cluster [DBG] pgmap v222: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:54:30.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:54:29 smithi089 bash[31372]: audit 2024-07-15T06:54:29.197615+0000 mon.a (mon.0) 943 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:54:30.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:54:29 smithi134 bash[24068]: cluster 2024-07-15T06:54:28.245899+0000 mgr.y (mgr.24272) 251 : cluster [DBG] pgmap v222: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:54:30.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:54:29 smithi134 bash[24068]: audit 2024-07-15T06:54:29.197615+0000 mon.a (mon.0) 943 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:54:31.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:54:30 smithi089 bash[21793]: audit 2024-07-15T06:54:30.248162+0000 mon.a (mon.0) 944 : audit [DBG] from='client.? 172.21.15.89:0/765575436' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:54:31.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:54:30 smithi089 bash[31372]: audit 2024-07-15T06:54:30.248162+0000 mon.a (mon.0) 944 : audit [DBG] from='client.? 172.21.15.89:0/765575436' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:54:31.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:54:30 smithi134 bash[24068]: audit 2024-07-15T06:54:30.248162+0000 mon.a (mon.0) 944 : audit [DBG] from='client.? 172.21.15.89:0/765575436' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:54:32.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:54:31 smithi089 bash[21793]: cluster 2024-07-15T06:54:30.246874+0000 mgr.y (mgr.24272) 252 : cluster [DBG] pgmap v223: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:54:32.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:54:31 smithi089 bash[31372]: cluster 2024-07-15T06:54:30.246874+0000 mgr.y (mgr.24272) 252 : cluster [DBG] pgmap v223: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:54:32.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:54:31 smithi134 bash[24068]: cluster 2024-07-15T06:54:30.246874+0000 mgr.y (mgr.24272) 252 : cluster [DBG] pgmap v223: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:54:33.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:54:32 smithi089 bash[21793]: audit 2024-07-15T06:54:32.616283+0000 mon.a (mon.0) 945 : audit [DBG] from='client.? 172.21.15.89:0/2863832351' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:54:33.119 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:54:32 smithi089 bash[22045]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:54:32] "GET /metrics HTTP/1.1" 200 239933 "" "Prometheus/2.43.0" 2024-07-15T06:54:33.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:54:32 smithi089 bash[31372]: audit 2024-07-15T06:54:32.616283+0000 mon.a (mon.0) 945 : audit [DBG] from='client.? 172.21.15.89:0/2863832351' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:54:33.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:54:32 smithi134 bash[24068]: audit 2024-07-15T06:54:32.616283+0000 mon.a (mon.0) 945 : audit [DBG] from='client.? 172.21.15.89:0/2863832351' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:54:34.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:54:33 smithi089 bash[21793]: cluster 2024-07-15T06:54:32.247500+0000 mgr.y (mgr.24272) 253 : cluster [DBG] pgmap v224: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:54:34.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:54:33 smithi089 bash[31372]: cluster 2024-07-15T06:54:32.247500+0000 mgr.y (mgr.24272) 253 : cluster [DBG] pgmap v224: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:54:34.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:54:33 smithi134 bash[24068]: cluster 2024-07-15T06:54:32.247500+0000 mgr.y (mgr.24272) 253 : cluster [DBG] pgmap v224: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:54:35.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:54:34 smithi134 bash[24068]: cluster 2024-07-15T06:54:34.248114+0000 mgr.y (mgr.24272) 254 : cluster [DBG] pgmap v225: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:54:35.369 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:54:34 smithi089 bash[21793]: cluster 2024-07-15T06:54:34.248114+0000 mgr.y (mgr.24272) 254 : cluster [DBG] pgmap v225: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:54:35.369 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:54:34 smithi089 bash[31372]: cluster 2024-07-15T06:54:34.248114+0000 mgr.y (mgr.24272) 254 : cluster [DBG] pgmap v225: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:54:36.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:54:35 smithi134 bash[24068]: audit 2024-07-15T06:54:34.969443+0000 mon.a (mon.0) 946 : audit [DBG] from='client.? 172.21.15.89:0/3416230563' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:54:36.368 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:54:35 smithi089 bash[21793]: audit 2024-07-15T06:54:34.969443+0000 mon.a (mon.0) 946 : audit [DBG] from='client.? 172.21.15.89:0/3416230563' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:54:36.369 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:54:35 smithi089 bash[31372]: audit 2024-07-15T06:54:34.969443+0000 mon.a (mon.0) 946 : audit [DBG] from='client.? 172.21.15.89:0/3416230563' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:54:37.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:54:36 smithi134 bash[24068]: cluster 2024-07-15T06:54:36.249196+0000 mgr.y (mgr.24272) 255 : cluster [DBG] pgmap v226: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:54:37.304 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:54:36 smithi134 bash[25019]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:54:36] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-07-15T06:54:37.368 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:54:36 smithi089 bash[31372]: cluster 2024-07-15T06:54:36.249196+0000 mgr.y (mgr.24272) 255 : cluster [DBG] pgmap v226: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:54:37.369 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:54:36 smithi089 bash[21793]: cluster 2024-07-15T06:54:36.249196+0000 mgr.y (mgr.24272) 255 : cluster [DBG] pgmap v226: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:54:38.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:54:37 smithi134 bash[24068]: audit 2024-07-15T06:54:37.332987+0000 mon.c (mon.2) 38 : audit [DBG] from='client.? 172.21.15.89:0/4205579286' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:54:38.368 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:54:37 smithi089 bash[21793]: audit 2024-07-15T06:54:37.332987+0000 mon.c (mon.2) 38 : audit [DBG] from='client.? 172.21.15.89:0/4205579286' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:54:38.369 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:54:37 smithi089 bash[31372]: audit 2024-07-15T06:54:37.332987+0000 mon.c (mon.2) 38 : audit [DBG] from='client.? 172.21.15.89:0/4205579286' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:54:39.220 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:54:38 smithi134 bash[24068]: cluster 2024-07-15T06:54:38.249805+0000 mgr.y (mgr.24272) 256 : cluster [DBG] pgmap v227: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:54:39.368 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:54:38 smithi089 bash[31372]: cluster 2024-07-15T06:54:38.249805+0000 mgr.y (mgr.24272) 256 : cluster [DBG] pgmap v227: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:54:39.369 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:54:38 smithi089 bash[21793]: cluster 2024-07-15T06:54:38.249805+0000 mgr.y (mgr.24272) 256 : cluster [DBG] pgmap v227: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:54:40.263 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:54:39 smithi134 bash[24068]: audit 2024-07-15T06:54:39.673594+0000 mon.a (mon.0) 947 : audit [DBG] from='client.? 172.21.15.89:0/2377534193' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:54:40.369 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:54:39 smithi089 bash[21793]: audit 2024-07-15T06:54:39.673594+0000 mon.a (mon.0) 947 : audit [DBG] from='client.? 172.21.15.89:0/2377534193' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:54:40.369 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:54:39 smithi089 bash[31372]: audit 2024-07-15T06:54:39.673594+0000 mon.a (mon.0) 947 : audit [DBG] from='client.? 172.21.15.89:0/2377534193' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:54:41.285 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:54:40 smithi134 bash[24068]: cephadm 2024-07-15T06:54:39.941898+0000 mgr.y (mgr.24272) 257 : cephadm [INF] Upgrade: Updating mgr.x 2024-07-15T06:54:41.285 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:54:40 smithi134 bash[24068]: audit 2024-07-15T06:54:39.950378+0000 mon.a (mon.0) 948 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:54:41.285 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:54:40 smithi134 bash[24068]: audit 2024-07-15T06:54:39.952341+0000 mon.a (mon.0) 949 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-07-15T06:54:41.286 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:54:40 smithi134 bash[24068]: audit 2024-07-15T06:54:39.954615+0000 mon.a (mon.0) 950 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-07-15T06:54:41.286 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:54:40 smithi134 bash[24068]: audit 2024-07-15T06:54:39.956385+0000 mon.a (mon.0) 951 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:54:41.286 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:54:40 smithi134 bash[24068]: cephadm 2024-07-15T06:54:39.958145+0000 mgr.y (mgr.24272) 258 : cephadm [INF] Deploying daemon mgr.x on smithi134 2024-07-15T06:54:41.286 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:54:40 smithi134 bash[24068]: cluster 2024-07-15T06:54:40.250840+0000 mgr.y (mgr.24272) 259 : cluster [DBG] pgmap v228: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:54:41.369 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:54:40 smithi089 bash[21793]: cephadm 2024-07-15T06:54:39.941898+0000 mgr.y (mgr.24272) 257 : cephadm [INF] Upgrade: Updating mgr.x 2024-07-15T06:54:41.369 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:54:40 smithi089 bash[21793]: audit 2024-07-15T06:54:39.950378+0000 mon.a (mon.0) 948 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:54:41.369 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:54:40 smithi089 bash[21793]: audit 2024-07-15T06:54:39.952341+0000 mon.a (mon.0) 949 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-07-15T06:54:41.369 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:54:40 smithi089 bash[21793]: audit 2024-07-15T06:54:39.954615+0000 mon.a (mon.0) 950 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-07-15T06:54:41.369 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:54:40 smithi089 bash[21793]: audit 2024-07-15T06:54:39.956385+0000 mon.a (mon.0) 951 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:54:41.369 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:54:40 smithi089 bash[21793]: cephadm 2024-07-15T06:54:39.958145+0000 mgr.y (mgr.24272) 258 : cephadm [INF] Deploying daemon mgr.x on smithi134 2024-07-15T06:54:41.369 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:54:40 smithi089 bash[21793]: cluster 2024-07-15T06:54:40.250840+0000 mgr.y (mgr.24272) 259 : cluster [DBG] pgmap v228: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:54:41.369 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:54:40 smithi089 bash[31372]: cephadm 2024-07-15T06:54:39.941898+0000 mgr.y (mgr.24272) 257 : cephadm [INF] Upgrade: Updating mgr.x 2024-07-15T06:54:41.369 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:54:40 smithi089 bash[31372]: audit 2024-07-15T06:54:39.950378+0000 mon.a (mon.0) 948 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:54:41.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:54:40 smithi089 bash[31372]: audit 2024-07-15T06:54:39.952341+0000 mon.a (mon.0) 949 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-07-15T06:54:41.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:54:40 smithi089 bash[31372]: audit 2024-07-15T06:54:39.954615+0000 mon.a (mon.0) 950 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-07-15T06:54:41.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:54:40 smithi089 bash[31372]: audit 2024-07-15T06:54:39.956385+0000 mon.a (mon.0) 951 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:54:41.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:54:40 smithi089 bash[31372]: cephadm 2024-07-15T06:54:39.958145+0000 mgr.y (mgr.24272) 258 : cephadm [INF] Deploying daemon mgr.x on smithi134 2024-07-15T06:54:41.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:54:40 smithi089 bash[31372]: cluster 2024-07-15T06:54:40.250840+0000 mgr.y (mgr.24272) 259 : cluster [DBG] pgmap v228: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:54:42.361 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:54:42 smithi134 bash[24068]: audit 2024-07-15T06:54:42.022978+0000 mon.a (mon.0) 952 : audit [DBG] from='client.? 172.21.15.89:0/4067708787' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:54:42.362 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:54:42 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:54:42.362 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:54:42 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:54:42.362 INFO:journalctl@ceph.osd.4.smithi134.stdout:Jul 15 06:54:42 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:54:42.362 INFO:journalctl@ceph.osd.6.smithi134.stdout:Jul 15 06:54:42 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:54:42.363 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:54:42 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:54:42.363 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:54:42 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:54:42.363 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 06:54:42 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:54:42.364 INFO:journalctl@ceph.osd.5.smithi134.stdout:Jul 15 06:54:42 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:54:42.364 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:54:42 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:54:42.368 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:54:42 smithi089 bash[21793]: audit 2024-07-15T06:54:42.022978+0000 mon.a (mon.0) 952 : audit [DBG] from='client.? 172.21.15.89:0/4067708787' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:54:42.368 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:54:42 smithi089 bash[31372]: audit 2024-07-15T06:54:42.022978+0000 mon.a (mon.0) 952 : audit [DBG] from='client.? 172.21.15.89:0/4067708787' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:54:42.717 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:54:42 smithi134 systemd[1]: Stopping Ceph mgr.x for 75ba02bc-4274-11ef-bcad-c7b262605968... 2024-07-15T06:54:43.077 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:54:42 smithi089 bash[22045]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:54:42] "GET /metrics HTTP/1.1" 200 239933 "" "Prometheus/2.43.0" 2024-07-15T06:54:43.303 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:54:43 smithi134 bash[24068]: cluster 2024-07-15T06:54:42.251453+0000 mgr.y (mgr.24272) 260 : cluster [DBG] pgmap v229: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:54:43.304 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:54:43 smithi134 bash[52466]: ceph-75ba02bc-4274-11ef-bcad-c7b262605968-mgr-x 2024-07-15T06:54:43.304 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:54:43 smithi134 systemd[1]: ceph-75ba02bc-4274-11ef-bcad-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-07-15T06:54:43.304 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:54:43 smithi134 bash[52528]: Error response from daemon: No such container: ceph-75ba02bc-4274-11ef-bcad-c7b262605968-mgr-x 2024-07-15T06:54:43.304 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:54:43 smithi134 systemd[1]: ceph-75ba02bc-4274-11ef-bcad-c7b262605968@mgr.x.service: Failed with result 'exit-code'. 2024-07-15T06:54:43.304 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:54:43 smithi134 systemd[1]: Stopped Ceph mgr.x for 75ba02bc-4274-11ef-bcad-c7b262605968. 2024-07-15T06:54:43.369 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:54:43 smithi089 bash[31372]: cluster 2024-07-15T06:54:42.251453+0000 mgr.y (mgr.24272) 260 : cluster [DBG] pgmap v229: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:54:43.369 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:54:43 smithi089 bash[21793]: cluster 2024-07-15T06:54:42.251453+0000 mgr.y (mgr.24272) 260 : cluster [DBG] pgmap v229: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:54:43.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:54:43 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:54:43.804 INFO:journalctl@ceph.osd.4.smithi134.stdout:Jul 15 06:54:43 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:54:43.804 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 06:54:43 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:54:43.805 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:54:43 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:54:43.805 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:54:43 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:54:43.805 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:54:43 smithi134 systemd[1]: Started Ceph mgr.x for 75ba02bc-4274-11ef-bcad-c7b262605968. 2024-07-15T06:54:43.805 INFO:journalctl@ceph.osd.5.smithi134.stdout:Jul 15 06:54:43 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:54:43.806 INFO:journalctl@ceph.osd.6.smithi134.stdout:Jul 15 06:54:43 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:54:43.806 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:54:43 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:54:43.806 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:54:43 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:54:43.868 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 06:54:43 smithi089 bash[58097]: ts=2024-07-15T06:54:43.525Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi134.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.134:8443: connect: connection refused" 2024-07-15T06:54:44.803 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:54:44 smithi134 bash[24068]: audit 2024-07-15T06:54:43.518885+0000 mon.a (mon.0) 953 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:54:44.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:54:44 smithi134 bash[24068]: audit 2024-07-15T06:54:43.530271+0000 mon.a (mon.0) 954 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:54:44.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:54:44 smithi134 bash[24068]: audit 2024-07-15T06:54:43.534068+0000 mon.a (mon.0) 955 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:54:44.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:54:44 smithi134 bash[24068]: audit 2024-07-15T06:54:44.197833+0000 mon.a (mon.0) 956 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:54:44.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:54:44 smithi134 bash[24068]: audit 2024-07-15T06:54:44.391857+0000 mon.c (mon.2) 39 : audit [DBG] from='client.? 172.21.15.89:0/472808157' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:54:44.869 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:54:44 smithi089 bash[31372]: audit 2024-07-15T06:54:43.518885+0000 mon.a (mon.0) 953 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:54:44.869 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:54:44 smithi089 bash[31372]: audit 2024-07-15T06:54:43.530271+0000 mon.a (mon.0) 954 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:54:44.869 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:54:44 smithi089 bash[31372]: audit 2024-07-15T06:54:43.534068+0000 mon.a (mon.0) 955 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:54:44.869 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:54:44 smithi089 bash[31372]: audit 2024-07-15T06:54:44.197833+0000 mon.a (mon.0) 956 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:54:44.869 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:54:44 smithi089 bash[31372]: audit 2024-07-15T06:54:44.391857+0000 mon.c (mon.2) 39 : audit [DBG] from='client.? 172.21.15.89:0/472808157' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:54:44.869 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:54:44 smithi089 bash[21793]: audit 2024-07-15T06:54:43.518885+0000 mon.a (mon.0) 953 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:54:44.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:54:44 smithi089 bash[21793]: audit 2024-07-15T06:54:43.530271+0000 mon.a (mon.0) 954 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:54:44.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:54:44 smithi089 bash[21793]: audit 2024-07-15T06:54:43.534068+0000 mon.a (mon.0) 955 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:54:44.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:54:44 smithi089 bash[21793]: audit 2024-07-15T06:54:44.197833+0000 mon.a (mon.0) 956 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:54:44.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:54:44 smithi089 bash[21793]: audit 2024-07-15T06:54:44.391857+0000 mon.c (mon.2) 39 : audit [DBG] from='client.? 172.21.15.89:0/472808157' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:54:45.304 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:54:44 smithi134 bash[52600]: debug 2024-07-15T06:54:44.828+0000 7faacf963640 1 -- 172.21.15.134:0/3185303174 <== mon.1 v2:172.21.15.134:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x565555b82d00 con 0x565555b75400 2024-07-15T06:54:45.304 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:54:44 smithi134 bash[52600]: debug 2024-07-15T06:54:44.940+0000 7faad21cc040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-07-15T06:54:45.803 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:54:45 smithi134 bash[24068]: cluster 2024-07-15T06:54:44.251858+0000 mgr.y (mgr.24272) 261 : cluster [DBG] pgmap v230: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:54:45.868 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:54:45 smithi089 bash[21793]: cluster 2024-07-15T06:54:44.251858+0000 mgr.y (mgr.24272) 261 : cluster [DBG] pgmap v230: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:54:45.869 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:54:45 smithi089 bash[31372]: cluster 2024-07-15T06:54:44.251858+0000 mgr.y (mgr.24272) 261 : cluster [DBG] pgmap v230: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:54:46.554 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:54:46 smithi134 bash[52600]: debug 2024-07-15T06:54:46.168+0000 7faad21cc040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-07-15T06:54:46.554 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:54:46 smithi134 bash[52600]: debug 2024-07-15T06:54:46.232+0000 7faad21cc040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-07-15T06:54:46.554 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:54:46 smithi134 bash[52600]: debug 2024-07-15T06:54:46.384+0000 7faad21cc040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-07-15T06:54:47.304 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:54:47 smithi134 bash[52600]: debug 2024-07-15T06:54:47.040+0000 7faad21cc040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-07-15T06:54:47.304 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:54:47 smithi134 bash[52600]: debug 2024-07-15T06:54:47.116+0000 7faad21cc040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-07-15T06:54:47.304 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:54:47 smithi134 bash[52600]: debug 2024-07-15T06:54:47.188+0000 7faad21cc040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-07-15T06:54:47.782 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:54:47 smithi134 bash[24068]: cluster 2024-07-15T06:54:46.252858+0000 mgr.y (mgr.24272) 262 : cluster [DBG] pgmap v231: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:54:47.782 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:54:47 smithi134 bash[24068]: audit 2024-07-15T06:54:46.743081+0000 mon.a (mon.0) 957 : audit [DBG] from='client.? 172.21.15.89:0/132585101' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:54:47.783 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:54:47 smithi134 bash[52600]: debug 2024-07-15T06:54:47.372+0000 7faad21cc040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-07-15T06:54:47.783 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:54:47 smithi134 bash[52600]: debug 2024-07-15T06:54:47.456+0000 7faad21cc040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-07-15T06:54:47.783 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:54:47 smithi134 bash[52600]: debug 2024-07-15T06:54:47.524+0000 7faad21cc040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-07-15T06:54:47.868 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:54:47 smithi089 bash[21793]: cluster 2024-07-15T06:54:46.252858+0000 mgr.y (mgr.24272) 262 : cluster [DBG] pgmap v231: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:54:47.868 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:54:47 smithi089 bash[21793]: audit 2024-07-15T06:54:46.743081+0000 mon.a (mon.0) 957 : audit [DBG] from='client.? 172.21.15.89:0/132585101' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:54:47.869 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:54:47 smithi089 bash[31372]: cluster 2024-07-15T06:54:46.252858+0000 mgr.y (mgr.24272) 262 : cluster [DBG] pgmap v231: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:54:47.869 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:54:47 smithi089 bash[31372]: audit 2024-07-15T06:54:46.743081+0000 mon.a (mon.0) 957 : audit [DBG] from='client.? 172.21.15.89:0/132585101' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:54:48.051 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:54:47 smithi134 bash[52600]: debug 2024-07-15T06:54:47.784+0000 7faad21cc040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-07-15T06:54:48.051 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:54:47 smithi134 bash[52600]: debug 2024-07-15T06:54:47.856+0000 7faad21cc040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-07-15T06:54:48.304 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:54:48 smithi134 bash[52600]: debug 2024-07-15T06:54:48.056+0000 7faad21cc040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-07-15T06:54:48.304 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:54:48 smithi134 bash[52600]: debug 2024-07-15T06:54:48.132+0000 7faad21cc040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-07-15T06:54:48.735 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:54:48 smithi134 bash[52600]: debug 2024-07-15T06:54:48.388+0000 7faad21cc040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-07-15T06:54:48.735 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:54:48 smithi134 bash[52600]: debug 2024-07-15T06:54:48.568+0000 7faad21cc040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-07-15T06:54:49.001 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:54:48 smithi134 bash[52600]: debug 2024-07-15T06:54:48.740+0000 7faad21cc040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-07-15T06:54:49.001 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:54:48 smithi134 bash[52600]: debug 2024-07-15T06:54:48.820+0000 7faad21cc040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-07-15T06:54:49.304 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:54:49 smithi134 bash[52600]: debug 2024-07-15T06:54:49.004+0000 7faad21cc040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-07-15T06:54:49.803 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:54:49 smithi134 bash[24068]: cluster 2024-07-15T06:54:48.253497+0000 mgr.y (mgr.24272) 263 : cluster [DBG] pgmap v232: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:54:49.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:54:49 smithi134 bash[24068]: audit 2024-07-15T06:54:48.910114+0000 mon.a (mon.0) 958 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:54:49.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:54:49 smithi134 bash[24068]: audit 2024-07-15T06:54:48.914031+0000 mon.a (mon.0) 959 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:54:49.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:54:49 smithi134 bash[24068]: audit 2024-07-15T06:54:49.100612+0000 mon.a (mon.0) 960 : audit [DBG] from='client.? 172.21.15.89:0/676397799' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:54:49.869 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:54:49 smithi089 bash[21793]: cluster 2024-07-15T06:54:48.253497+0000 mgr.y (mgr.24272) 263 : cluster [DBG] pgmap v232: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:54:49.869 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:54:49 smithi089 bash[21793]: audit 2024-07-15T06:54:48.910114+0000 mon.a (mon.0) 958 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:54:49.869 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:54:49 smithi089 bash[21793]: audit 2024-07-15T06:54:48.914031+0000 mon.a (mon.0) 959 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:54:49.869 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:54:49 smithi089 bash[21793]: audit 2024-07-15T06:54:49.100612+0000 mon.a (mon.0) 960 : audit [DBG] from='client.? 172.21.15.89:0/676397799' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:54:49.869 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:54:49 smithi089 bash[31372]: cluster 2024-07-15T06:54:48.253497+0000 mgr.y (mgr.24272) 263 : cluster [DBG] pgmap v232: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:54:49.869 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:54:49 smithi089 bash[31372]: audit 2024-07-15T06:54:48.910114+0000 mon.a (mon.0) 958 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:54:49.869 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:54:49 smithi089 bash[31372]: audit 2024-07-15T06:54:48.914031+0000 mon.a (mon.0) 959 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:54:49.869 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:54:49 smithi089 bash[31372]: audit 2024-07-15T06:54:49.100612+0000 mon.a (mon.0) 960 : audit [DBG] from='client.? 172.21.15.89:0/676397799' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:54:50.190 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:54:49 smithi134 bash[52600]: debug 2024-07-15T06:54:49.856+0000 7faad21cc040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-07-15T06:54:50.191 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:54:49 smithi134 bash[52600]: debug 2024-07-15T06:54:49.940+0000 7faad21cc040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-07-15T06:54:50.553 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:54:50 smithi134 bash[52600]: debug 2024-07-15T06:54:50.192+0000 7faad21cc040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-07-15T06:54:50.554 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:54:50 smithi134 bash[52600]: debug 2024-07-15T06:54:50.276+0000 7faad21cc040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-07-15T06:54:51.054 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:54:50 smithi134 bash[52600]: debug 2024-07-15T06:54:50.688+0000 7faad21cc040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-07-15T06:54:51.054 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:54:50 smithi134 bash[52600]: debug 2024-07-15T06:54:50.768+0000 7faad21cc040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-07-15T06:54:51.054 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:54:50 smithi134 bash[52600]: [15/Jul/2024:06:54:50] ENGINE Bus STARTING 2024-07-15T06:54:51.054 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:54:50 smithi134 bash[52600]: CherryPy Checker: 2024-07-15T06:54:51.054 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:54:50 smithi134 bash[52600]: The Application mounted at '' has an empty config. 2024-07-15T06:54:51.054 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:54:50 smithi134 bash[52600]: [15/Jul/2024:06:54:50] ENGINE Serving on http://:::9283 2024-07-15T06:54:51.054 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:54:50 smithi134 bash[52600]: [15/Jul/2024:06:54:50] ENGINE Bus STARTED 2024-07-15T06:54:51.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:54:51 smithi089 bash[21793]: cluster 2024-07-15T06:54:50.254525+0000 mgr.y (mgr.24272) 264 : cluster [DBG] pgmap v233: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:54:51.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:54:51 smithi089 bash[21793]: audit 2024-07-15T06:54:50.348644+0000 mon.a (mon.0) 961 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:54:51.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:54:51 smithi089 bash[21793]: audit 2024-07-15T06:54:50.358787+0000 mon.a (mon.0) 962 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:54:51.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:54:51 smithi089 bash[21793]: cluster 2024-07-15T06:54:50.773828+0000 mon.a (mon.0) 963 : cluster [DBG] Standby manager daemon x restarted 2024-07-15T06:54:51.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:54:51 smithi089 bash[21793]: cluster 2024-07-15T06:54:50.774076+0000 mon.a (mon.0) 964 : cluster [DBG] Standby manager daemon x started 2024-07-15T06:54:51.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:54:51 smithi089 bash[21793]: audit 2024-07-15T06:54:50.775757+0000 mon.b (mon.1) 30 : audit [DBG] from='mgr.? 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-07-15T06:54:51.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:54:51 smithi089 bash[21793]: audit 2024-07-15T06:54:50.776196+0000 mon.b (mon.1) 31 : audit [DBG] from='mgr.? 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-07-15T06:54:51.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:54:51 smithi089 bash[21793]: audit 2024-07-15T06:54:50.777691+0000 mon.b (mon.1) 32 : audit [DBG] from='mgr.? 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-07-15T06:54:51.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:54:51 smithi089 bash[21793]: audit 2024-07-15T06:54:50.778696+0000 mon.b (mon.1) 33 : audit [DBG] from='mgr.? 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-07-15T06:54:51.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:54:51 smithi089 bash[21793]: audit 2024-07-15T06:54:50.961187+0000 mon.a (mon.0) 965 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:54:51.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:54:51 smithi089 bash[21793]: audit 2024-07-15T06:54:50.974694+0000 mon.a (mon.0) 966 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:54:51.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:54:51 smithi089 bash[31372]: cluster 2024-07-15T06:54:50.254525+0000 mgr.y (mgr.24272) 264 : cluster [DBG] pgmap v233: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:54:51.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:54:51 smithi089 bash[31372]: audit 2024-07-15T06:54:50.348644+0000 mon.a (mon.0) 961 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:54:51.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:54:51 smithi089 bash[31372]: audit 2024-07-15T06:54:50.358787+0000 mon.a (mon.0) 962 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:54:51.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:54:51 smithi089 bash[31372]: cluster 2024-07-15T06:54:50.773828+0000 mon.a (mon.0) 963 : cluster [DBG] Standby manager daemon x restarted 2024-07-15T06:54:51.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:54:51 smithi089 bash[31372]: cluster 2024-07-15T06:54:50.774076+0000 mon.a (mon.0) 964 : cluster [DBG] Standby manager daemon x started 2024-07-15T06:54:51.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:54:51 smithi089 bash[31372]: audit 2024-07-15T06:54:50.775757+0000 mon.b (mon.1) 30 : audit [DBG] from='mgr.? 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-07-15T06:54:51.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:54:51 smithi089 bash[31372]: audit 2024-07-15T06:54:50.776196+0000 mon.b (mon.1) 31 : audit [DBG] from='mgr.? 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-07-15T06:54:51.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:54:51 smithi089 bash[31372]: audit 2024-07-15T06:54:50.777691+0000 mon.b (mon.1) 32 : audit [DBG] from='mgr.? 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-07-15T06:54:51.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:54:51 smithi089 bash[31372]: audit 2024-07-15T06:54:50.778696+0000 mon.b (mon.1) 33 : audit [DBG] from='mgr.? 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-07-15T06:54:51.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:54:51 smithi089 bash[31372]: audit 2024-07-15T06:54:50.961187+0000 mon.a (mon.0) 965 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:54:51.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:54:51 smithi089 bash[31372]: audit 2024-07-15T06:54:50.974694+0000 mon.a (mon.0) 966 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:54:51.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:54:51 smithi134 bash[24068]: cluster 2024-07-15T06:54:50.254525+0000 mgr.y (mgr.24272) 264 : cluster [DBG] pgmap v233: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:54:51.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:54:51 smithi134 bash[24068]: audit 2024-07-15T06:54:50.348644+0000 mon.a (mon.0) 961 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:54:51.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:54:51 smithi134 bash[24068]: audit 2024-07-15T06:54:50.358787+0000 mon.a (mon.0) 962 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:54:51.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:54:51 smithi134 bash[24068]: cluster 2024-07-15T06:54:50.773828+0000 mon.a (mon.0) 963 : cluster [DBG] Standby manager daemon x restarted 2024-07-15T06:54:51.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:54:51 smithi134 bash[24068]: cluster 2024-07-15T06:54:50.774076+0000 mon.a (mon.0) 964 : cluster [DBG] Standby manager daemon x started 2024-07-15T06:54:51.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:54:51 smithi134 bash[24068]: audit 2024-07-15T06:54:50.775757+0000 mon.b (mon.1) 30 : audit [DBG] from='mgr.? 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-07-15T06:54:51.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:54:51 smithi134 bash[24068]: audit 2024-07-15T06:54:50.776196+0000 mon.b (mon.1) 31 : audit [DBG] from='mgr.? 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-07-15T06:54:51.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:54:51 smithi134 bash[24068]: audit 2024-07-15T06:54:50.777691+0000 mon.b (mon.1) 32 : audit [DBG] from='mgr.? 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-07-15T06:54:51.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:54:51 smithi134 bash[24068]: audit 2024-07-15T06:54:50.778696+0000 mon.b (mon.1) 33 : audit [DBG] from='mgr.? 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-07-15T06:54:51.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:54:51 smithi134 bash[24068]: audit 2024-07-15T06:54:50.961187+0000 mon.a (mon.0) 965 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:54:51.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:54:51 smithi134 bash[24068]: audit 2024-07-15T06:54:50.974694+0000 mon.a (mon.0) 966 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:54:52.618 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:54:52 smithi089 bash[31372]: audit 2024-07-15T06:54:51.461551+0000 mon.c (mon.2) 40 : audit [DBG] from='client.? 172.21.15.89:0/3350180838' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:54:52.619 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:54:52 smithi089 bash[31372]: cluster 2024-07-15T06:54:51.553069+0000 mon.a (mon.0) 967 : cluster [DBG] mgrmap e21: y(active, since 7m), standbys: x 2024-07-15T06:54:52.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:54:52 smithi089 bash[21793]: audit 2024-07-15T06:54:51.461551+0000 mon.c (mon.2) 40 : audit [DBG] from='client.? 172.21.15.89:0/3350180838' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:54:52.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:54:52 smithi089 bash[21793]: cluster 2024-07-15T06:54:51.553069+0000 mon.a (mon.0) 967 : cluster [DBG] mgrmap e21: y(active, since 7m), standbys: x 2024-07-15T06:54:52.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:54:52 smithi134 bash[24068]: audit 2024-07-15T06:54:51.461551+0000 mon.c (mon.2) 40 : audit [DBG] from='client.? 172.21.15.89:0/3350180838' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:54:52.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:54:52 smithi134 bash[24068]: cluster 2024-07-15T06:54:51.553069+0000 mon.a (mon.0) 967 : cluster [DBG] mgrmap e21: y(active, since 7m), standbys: x 2024-07-15T06:54:53.118 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:54:52 smithi089 bash[22045]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:54:52] "GET /metrics HTTP/1.1" 200 239934 "" "Prometheus/2.43.0" 2024-07-15T06:54:53.803 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:54:53 smithi134 bash[24068]: cluster 2024-07-15T06:54:52.255156+0000 mgr.y (mgr.24272) 265 : cluster [DBG] pgmap v234: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:54:53.868 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:54:53 smithi089 bash[21793]: cluster 2024-07-15T06:54:52.255156+0000 mgr.y (mgr.24272) 265 : cluster [DBG] pgmap v234: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:54:53.869 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 06:54:53 smithi089 bash[58097]: ts=2024-07-15T06:54:53.522Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi134.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.134:8443: connect: connection refused" 2024-07-15T06:54:53.869 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:54:53 smithi089 bash[31372]: cluster 2024-07-15T06:54:52.255156+0000 mgr.y (mgr.24272) 265 : cluster [DBG] pgmap v234: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:54:54.803 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:54:54 smithi134 bash[24068]: audit 2024-07-15T06:54:53.831530+0000 mon.a (mon.0) 968 : audit [DBG] from='client.? 172.21.15.89:0/1793814914' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:54:54.869 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:54:54 smithi089 bash[21793]: audit 2024-07-15T06:54:53.831530+0000 mon.a (mon.0) 968 : audit [DBG] from='client.? 172.21.15.89:0/1793814914' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:54:54.869 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:54:54 smithi089 bash[31372]: audit 2024-07-15T06:54:53.831530+0000 mon.a (mon.0) 968 : audit [DBG] from='client.? 172.21.15.89:0/1793814914' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:54:55.865 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:54:55 smithi134 bash[24068]: cluster 2024-07-15T06:54:54.255767+0000 mgr.y (mgr.24272) 266 : cluster [DBG] pgmap v235: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:54:55.868 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:54:55 smithi089 bash[21793]: cluster 2024-07-15T06:54:54.255767+0000 mgr.y (mgr.24272) 266 : cluster [DBG] pgmap v235: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:54:55.869 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:54:55 smithi089 bash[31372]: cluster 2024-07-15T06:54:54.255767+0000 mgr.y (mgr.24272) 266 : cluster [DBG] pgmap v235: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:54:56.869 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:54:56 smithi089 bash[21793]: audit 2024-07-15T06:54:56.187629+0000 mon.c (mon.2) 41 : audit [DBG] from='client.? 172.21.15.89:0/2624535405' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:54:56.869 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:54:56 smithi089 bash[31372]: audit 2024-07-15T06:54:56.187629+0000 mon.c (mon.2) 41 : audit [DBG] from='client.? 172.21.15.89:0/2624535405' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:54:56.908 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:54:56 smithi134 bash[24068]: audit 2024-07-15T06:54:56.187629+0000 mon.c (mon.2) 41 : audit [DBG] from='client.? 172.21.15.89:0/2624535405' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:54:57.274 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:54:56 smithi134 bash[52600]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:54:56] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-07-15T06:54:57.824 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:54:57 smithi134 bash[24068]: cluster 2024-07-15T06:54:56.256469+0000 mgr.y (mgr.24272) 267 : cluster [DBG] pgmap v236: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:54:57.868 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:54:57 smithi089 bash[31372]: cluster 2024-07-15T06:54:56.256469+0000 mgr.y (mgr.24272) 267 : cluster [DBG] pgmap v236: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:54:57.869 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:54:57 smithi089 bash[21793]: cluster 2024-07-15T06:54:56.256469+0000 mgr.y (mgr.24272) 267 : cluster [DBG] pgmap v236: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:54:58.869 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:54:58 smithi089 bash[21793]: audit 2024-07-15T06:54:58.547026+0000 mon.c (mon.2) 42 : audit [DBG] from='client.? 172.21.15.89:0/4077199431' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:54:58.869 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:54:58 smithi089 bash[31372]: audit 2024-07-15T06:54:58.547026+0000 mon.c (mon.2) 42 : audit [DBG] from='client.? 172.21.15.89:0/4077199431' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:54:59.053 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:54:58 smithi134 bash[24068]: audit 2024-07-15T06:54:58.547026+0000 mon.c (mon.2) 42 : audit [DBG] from='client.? 172.21.15.89:0/4077199431' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:54:59.869 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:54:59 smithi089 bash[21793]: cluster 2024-07-15T06:54:58.257021+0000 mgr.y (mgr.24272) 268 : cluster [DBG] pgmap v237: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:54:59.869 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:54:59 smithi089 bash[21793]: audit 2024-07-15T06:54:59.198535+0000 mon.a (mon.0) 969 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:54:59.869 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:54:59 smithi089 bash[31372]: cluster 2024-07-15T06:54:58.257021+0000 mgr.y (mgr.24272) 268 : cluster [DBG] pgmap v237: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:54:59.869 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:54:59 smithi089 bash[31372]: audit 2024-07-15T06:54:59.198535+0000 mon.a (mon.0) 969 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:54:59.899 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:54:59 smithi134 bash[24068]: cluster 2024-07-15T06:54:58.257021+0000 mgr.y (mgr.24272) 268 : cluster [DBG] pgmap v237: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:54:59.899 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:54:59 smithi134 bash[24068]: audit 2024-07-15T06:54:59.198535+0000 mon.a (mon.0) 969 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:55:00.804 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:55:00 smithi134 bash[52600]: [15/Jul/2024:06:55:00] ENGINE Bus STOPPING 2024-07-15T06:55:00.869 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:55:00 smithi089 bash[22045]: debug 2024-07-15T06:55:00.499+0000 7fc70c13e640 -1 mgr handle_mgr_map I was active but no longer am 2024-07-15T06:55:00.869 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:55:00 smithi089 bash[22045]: ignoring --setuser ceph since I am not root 2024-07-15T06:55:00.869 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:55:00 smithi089 bash[22045]: ignoring --setgroup ceph since I am not root 2024-07-15T06:55:00.869 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:55:00 smithi089 bash[22045]: debug 2024-07-15T06:55:00.555+0000 7f30c0c94640 1 -- 172.21.15.89:0/4281500779 <== mon.0 v2:172.21.15.89:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55ae61cec9c0 con 0x55ae61cf9000 2024-07-15T06:55:00.869 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:55:00 smithi089 bash[22045]: debug 2024-07-15T06:55:00.667+0000 7f30c34eeec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-07-15T06:55:01.297 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:55:00 smithi134 bash[52600]: [15/Jul/2024:06:55:00] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-07-15T06:55:01.297 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:55:00 smithi134 bash[52600]: [15/Jul/2024:06:55:00] ENGINE Bus STOPPED 2024-07-15T06:55:01.297 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:55:01 smithi134 bash[52600]: [15/Jul/2024:06:55:01] ENGINE Bus STARTING 2024-07-15T06:55:01.297 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:55:01 smithi134 bash[52600]: debug 2024-07-15T06:55:01.119+0000 7faa387ac640 -1 client.0 error registering admin socket command: (17) File exists 2024-07-15T06:55:01.297 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:55:01 smithi134 bash[52600]: debug 2024-07-15T06:55:01.119+0000 7faa387ac640 -1 client.0 error registering admin socket command: (17) File exists 2024-07-15T06:55:01.297 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:55:01 smithi134 bash[52600]: debug 2024-07-15T06:55:01.119+0000 7faa387ac640 -1 client.0 error registering admin socket command: (17) File exists 2024-07-15T06:55:01.297 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:55:01 smithi134 bash[52600]: debug 2024-07-15T06:55:01.119+0000 7faa387ac640 -1 client.0 error registering admin socket command: (17) File exists 2024-07-15T06:55:01.297 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:55:01 smithi134 bash[52600]: debug 2024-07-15T06:55:01.119+0000 7faa387ac640 -1 client.0 error registering admin socket command: (17) File exists 2024-07-15T06:55:01.297 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:55:01 smithi134 bash[52600]: debug 2024-07-15T06:55:01.123+0000 7faa3cfb5640 -1 client.0 error registering admin socket command: (17) File exists 2024-07-15T06:55:01.298 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:55:01 smithi134 bash[52600]: debug 2024-07-15T06:55:01.123+0000 7faa3cfb5640 -1 client.0 error registering admin socket command: (17) File exists 2024-07-15T06:55:01.298 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:55:01 smithi134 bash[52600]: debug 2024-07-15T06:55:01.123+0000 7faa3cfb5640 -1 client.0 error registering admin socket command: (17) File exists 2024-07-15T06:55:01.298 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:55:01 smithi134 bash[52600]: debug 2024-07-15T06:55:01.123+0000 7faa3cfb5640 -1 client.0 error registering admin socket command: (17) File exists 2024-07-15T06:55:01.298 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:55:01 smithi134 bash[52600]: debug 2024-07-15T06:55:01.123+0000 7faa3cfb5640 -1 client.0 error registering admin socket command: (17) File exists 2024-07-15T06:55:01.298 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:55:01 smithi134 bash[52600]: [15/Jul/2024:06:55:01] ENGINE Serving on http://:::9283 2024-07-15T06:55:01.298 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:55:01 smithi134 bash[52600]: [15/Jul/2024:06:55:01] ENGINE Bus STARTED 2024-07-15T06:55:01.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:01 smithi134 bash[24068]: audit 2024-07-15T06:55:00.385644+0000 mon.a (mon.0) 970 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:55:01.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:01 smithi134 bash[24068]: audit 2024-07-15T06:55:00.395763+0000 mon.a (mon.0) 971 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:55:01.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:01 smithi134 bash[24068]: audit 2024-07-15T06:55:00.397889+0000 mon.a (mon.0) 972 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:55:01.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:01 smithi134 bash[24068]: audit 2024-07-15T06:55:00.399767+0000 mon.a (mon.0) 973 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:55:01.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:01 smithi134 bash[24068]: audit 2024-07-15T06:55:00.410025+0000 mon.a (mon.0) 974 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:55:01.557 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:01 smithi134 bash[24068]: audit 2024-07-15T06:55:00.480040+0000 mon.a (mon.0) 975 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:55:01.557 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:01 smithi134 bash[24068]: audit 2024-07-15T06:55:00.492466+0000 mon.a (mon.0) 976 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-07-15T06:55:01.557 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:01 smithi134 bash[24068]: cluster 2024-07-15T06:55:00.493245+0000 mon.a (mon.0) 977 : cluster [INF] Activating manager daemon x 2024-07-15T06:55:01.557 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:01 smithi134 bash[24068]: cluster 2024-07-15T06:55:00.504997+0000 mon.a (mon.0) 978 : cluster [DBG] osdmap e77: 8 total, 8 up, 8 in 2024-07-15T06:55:01.557 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:01 smithi134 bash[24068]: audit 2024-07-15T06:55:00.505327+0000 mon.a (mon.0) 979 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-07-15T06:55:01.557 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:01 smithi134 bash[24068]: cluster 2024-07-15T06:55:00.505382+0000 mon.a (mon.0) 980 : cluster [DBG] mgrmap e22: x(active, starting, since 0.0121433s) 2024-07-15T06:55:01.558 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:01 smithi134 bash[24068]: audit 2024-07-15T06:55:00.509196+0000 mon.b (mon.1) 34 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-07-15T06:55:01.558 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:01 smithi134 bash[24068]: audit 2024-07-15T06:55:00.509629+0000 mon.b (mon.1) 35 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-15T06:55:01.558 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:01 smithi134 bash[24068]: audit 2024-07-15T06:55:00.510039+0000 mon.b (mon.1) 36 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-15T06:55:01.558 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:01 smithi134 bash[24068]: audit 2024-07-15T06:55:00.510442+0000 mon.b (mon.1) 37 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi134.wugtzh"}]: dispatch 2024-07-15T06:55:01.558 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:01 smithi134 bash[24068]: audit 2024-07-15T06:55:00.510929+0000 mon.b (mon.1) 38 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi089.cczuoz"}]: dispatch 2024-07-15T06:55:01.558 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:01 smithi134 bash[24068]: audit 2024-07-15T06:55:00.511681+0000 mon.b (mon.1) 39 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-07-15T06:55:01.558 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:01 smithi134 bash[24068]: audit 2024-07-15T06:55:00.512189+0000 mon.b (mon.1) 40 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-07-15T06:55:01.558 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:01 smithi134 bash[24068]: audit 2024-07-15T06:55:00.512575+0000 mon.b (mon.1) 41 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-07-15T06:55:01.558 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:01 smithi134 bash[24068]: audit 2024-07-15T06:55:00.512815+0000 mon.b (mon.1) 42 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-07-15T06:55:01.558 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:01 smithi134 bash[24068]: audit 2024-07-15T06:55:00.513257+0000 mon.b (mon.1) 43 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-07-15T06:55:01.558 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:01 smithi134 bash[24068]: audit 2024-07-15T06:55:00.513664+0000 mon.b (mon.1) 44 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-07-15T06:55:01.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:01 smithi134 bash[24068]: audit 2024-07-15T06:55:00.514179+0000 mon.b (mon.1) 45 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-07-15T06:55:01.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:01 smithi134 bash[24068]: audit 2024-07-15T06:55:00.514708+0000 mon.b (mon.1) 46 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-07-15T06:55:01.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:01 smithi134 bash[24068]: audit 2024-07-15T06:55:00.515238+0000 mon.b (mon.1) 47 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-07-15T06:55:01.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:01 smithi134 bash[24068]: audit 2024-07-15T06:55:00.516340+0000 mon.b (mon.1) 48 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-07-15T06:55:01.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:01 smithi134 bash[24068]: audit 2024-07-15T06:55:00.516885+0000 mon.b (mon.1) 49 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-07-15T06:55:01.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:01 smithi134 bash[24068]: audit 2024-07-15T06:55:00.517600+0000 mon.b (mon.1) 50 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-07-15T06:55:01.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:01 smithi134 bash[24068]: audit 2024-07-15T06:55:00.891838+0000 mon.a (mon.0) 981 : audit [DBG] from='client.? 172.21.15.89:0/4199570889' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:55:01.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:01 smithi134 bash[24068]: cluster 2024-07-15T06:55:01.025511+0000 mon.a (mon.0) 982 : cluster [INF] Manager daemon x is now available 2024-07-15T06:55:01.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:01 smithi134 bash[24068]: audit 2024-07-15T06:55:01.053755+0000 mon.b (mon.1) 51 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:55:01.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:01 smithi134 bash[24068]: audit 2024-07-15T06:55:01.080090+0000 mon.b (mon.1) 52 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:55:01.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:01 smithi134 bash[24068]: audit 2024-07-15T06:55:01.097903+0000 mon.b (mon.1) 53 : audit [INF] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-07-15T06:55:01.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:01 smithi134 bash[24068]: audit 2024-07-15T06:55:01.098559+0000 mon.a (mon.0) 983 : audit [INF] from='mgr.24544 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-07-15T06:55:01.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:01 smithi134 bash[24068]: audit 2024-07-15T06:55:01.112244+0000 mon.b (mon.1) 54 : audit [INF] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-07-15T06:55:01.560 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:01 smithi134 bash[24068]: audit 2024-07-15T06:55:01.112705+0000 mon.a (mon.0) 984 : audit [INF] from='mgr.24544 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-07-15T06:55:01.618 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:01 smithi089 bash[21793]: audit 2024-07-15T06:55:00.385644+0000 mon.a (mon.0) 970 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:55:01.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:01 smithi089 bash[21793]: audit 2024-07-15T06:55:00.395763+0000 mon.a (mon.0) 971 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:55:01.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:01 smithi089 bash[21793]: audit 2024-07-15T06:55:00.397889+0000 mon.a (mon.0) 972 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:55:01.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:01 smithi089 bash[21793]: audit 2024-07-15T06:55:00.399767+0000 mon.a (mon.0) 973 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:55:01.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:01 smithi089 bash[21793]: audit 2024-07-15T06:55:00.410025+0000 mon.a (mon.0) 974 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:55:01.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:01 smithi089 bash[21793]: audit 2024-07-15T06:55:00.480040+0000 mon.a (mon.0) 975 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:55:01.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:01 smithi089 bash[21793]: audit 2024-07-15T06:55:00.492466+0000 mon.a (mon.0) 976 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-07-15T06:55:01.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:01 smithi089 bash[21793]: cluster 2024-07-15T06:55:00.493245+0000 mon.a (mon.0) 977 : cluster [INF] Activating manager daemon x 2024-07-15T06:55:01.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:01 smithi089 bash[21793]: cluster 2024-07-15T06:55:00.504997+0000 mon.a (mon.0) 978 : cluster [DBG] osdmap e77: 8 total, 8 up, 8 in 2024-07-15T06:55:01.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:01 smithi089 bash[21793]: audit 2024-07-15T06:55:00.505327+0000 mon.a (mon.0) 979 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-07-15T06:55:01.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:01 smithi089 bash[21793]: cluster 2024-07-15T06:55:00.505382+0000 mon.a (mon.0) 980 : cluster [DBG] mgrmap e22: x(active, starting, since 0.0121433s) 2024-07-15T06:55:01.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:01 smithi089 bash[21793]: audit 2024-07-15T06:55:00.509196+0000 mon.b (mon.1) 34 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-07-15T06:55:01.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:01 smithi089 bash[21793]: audit 2024-07-15T06:55:00.509629+0000 mon.b (mon.1) 35 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-15T06:55:01.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:01 smithi089 bash[21793]: audit 2024-07-15T06:55:00.510039+0000 mon.b (mon.1) 36 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-15T06:55:01.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:01 smithi089 bash[21793]: audit 2024-07-15T06:55:00.510442+0000 mon.b (mon.1) 37 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi134.wugtzh"}]: dispatch 2024-07-15T06:55:01.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:01 smithi089 bash[21793]: audit 2024-07-15T06:55:00.510929+0000 mon.b (mon.1) 38 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi089.cczuoz"}]: dispatch 2024-07-15T06:55:01.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:01 smithi089 bash[21793]: audit 2024-07-15T06:55:00.511681+0000 mon.b (mon.1) 39 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-07-15T06:55:01.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:01 smithi089 bash[21793]: audit 2024-07-15T06:55:00.512189+0000 mon.b (mon.1) 40 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-07-15T06:55:01.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:01 smithi089 bash[21793]: audit 2024-07-15T06:55:00.512575+0000 mon.b (mon.1) 41 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-07-15T06:55:01.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:01 smithi089 bash[31372]: audit 2024-07-15T06:55:00.385644+0000 mon.a (mon.0) 970 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:55:01.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:01 smithi089 bash[31372]: audit 2024-07-15T06:55:00.395763+0000 mon.a (mon.0) 971 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:55:01.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:01 smithi089 bash[31372]: audit 2024-07-15T06:55:00.397889+0000 mon.a (mon.0) 972 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:55:01.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:01 smithi089 bash[31372]: audit 2024-07-15T06:55:00.399767+0000 mon.a (mon.0) 973 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:55:01.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:01 smithi089 bash[31372]: audit 2024-07-15T06:55:00.410025+0000 mon.a (mon.0) 974 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' 2024-07-15T06:55:01.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:01 smithi089 bash[31372]: audit 2024-07-15T06:55:00.480040+0000 mon.a (mon.0) 975 : audit [DBG] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:55:01.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:01 smithi089 bash[31372]: audit 2024-07-15T06:55:00.492466+0000 mon.a (mon.0) 976 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-07-15T06:55:01.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:01 smithi089 bash[31372]: cluster 2024-07-15T06:55:00.493245+0000 mon.a (mon.0) 977 : cluster [INF] Activating manager daemon x 2024-07-15T06:55:01.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:01 smithi089 bash[31372]: cluster 2024-07-15T06:55:00.504997+0000 mon.a (mon.0) 978 : cluster [DBG] osdmap e77: 8 total, 8 up, 8 in 2024-07-15T06:55:01.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:01 smithi089 bash[31372]: audit 2024-07-15T06:55:00.505327+0000 mon.a (mon.0) 979 : audit [INF] from='mgr.24272 172.21.15.89:0/1807485518' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-07-15T06:55:01.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:01 smithi089 bash[31372]: cluster 2024-07-15T06:55:00.505382+0000 mon.a (mon.0) 980 : cluster [DBG] mgrmap e22: x(active, starting, since 0.0121433s) 2024-07-15T06:55:01.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:01 smithi089 bash[31372]: audit 2024-07-15T06:55:00.509196+0000 mon.b (mon.1) 34 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-07-15T06:55:01.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:01 smithi089 bash[31372]: audit 2024-07-15T06:55:00.509629+0000 mon.b (mon.1) 35 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-15T06:55:01.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:01 smithi089 bash[31372]: audit 2024-07-15T06:55:00.510039+0000 mon.b (mon.1) 36 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-15T06:55:01.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:01 smithi089 bash[31372]: audit 2024-07-15T06:55:00.510442+0000 mon.b (mon.1) 37 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi134.wugtzh"}]: dispatch 2024-07-15T06:55:01.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:01 smithi089 bash[31372]: audit 2024-07-15T06:55:00.510929+0000 mon.b (mon.1) 38 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi089.cczuoz"}]: dispatch 2024-07-15T06:55:01.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:01 smithi089 bash[31372]: audit 2024-07-15T06:55:00.511681+0000 mon.b (mon.1) 39 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-07-15T06:55:01.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:01 smithi089 bash[31372]: audit 2024-07-15T06:55:00.512189+0000 mon.b (mon.1) 40 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-07-15T06:55:01.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:01 smithi089 bash[31372]: audit 2024-07-15T06:55:00.512575+0000 mon.b (mon.1) 41 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-07-15T06:55:01.622 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:01 smithi089 bash[21793]: audit 2024-07-15T06:55:00.512815+0000 mon.b (mon.1) 42 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-07-15T06:55:01.622 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:01 smithi089 bash[21793]: audit 2024-07-15T06:55:00.513257+0000 mon.b (mon.1) 43 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-07-15T06:55:01.622 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:01 smithi089 bash[21793]: audit 2024-07-15T06:55:00.513664+0000 mon.b (mon.1) 44 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-07-15T06:55:01.622 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:01 smithi089 bash[21793]: audit 2024-07-15T06:55:00.514179+0000 mon.b (mon.1) 45 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-07-15T06:55:01.622 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:01 smithi089 bash[21793]: audit 2024-07-15T06:55:00.514708+0000 mon.b (mon.1) 46 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-07-15T06:55:01.622 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:01 smithi089 bash[21793]: audit 2024-07-15T06:55:00.515238+0000 mon.b (mon.1) 47 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-07-15T06:55:01.622 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:01 smithi089 bash[21793]: audit 2024-07-15T06:55:00.516340+0000 mon.b (mon.1) 48 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-07-15T06:55:01.622 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:01 smithi089 bash[21793]: audit 2024-07-15T06:55:00.516885+0000 mon.b (mon.1) 49 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-07-15T06:55:01.623 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:01 smithi089 bash[21793]: audit 2024-07-15T06:55:00.517600+0000 mon.b (mon.1) 50 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-07-15T06:55:01.623 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:01 smithi089 bash[21793]: audit 2024-07-15T06:55:00.891838+0000 mon.a (mon.0) 981 : audit [DBG] from='client.? 172.21.15.89:0/4199570889' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:55:01.623 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:01 smithi089 bash[21793]: cluster 2024-07-15T06:55:01.025511+0000 mon.a (mon.0) 982 : cluster [INF] Manager daemon x is now available 2024-07-15T06:55:01.623 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:01 smithi089 bash[21793]: audit 2024-07-15T06:55:01.053755+0000 mon.b (mon.1) 51 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:55:01.623 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:01 smithi089 bash[21793]: audit 2024-07-15T06:55:01.080090+0000 mon.b (mon.1) 52 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:55:01.623 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:01 smithi089 bash[21793]: audit 2024-07-15T06:55:01.097903+0000 mon.b (mon.1) 53 : audit [INF] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-07-15T06:55:01.623 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:01 smithi089 bash[21793]: audit 2024-07-15T06:55:01.098559+0000 mon.a (mon.0) 983 : audit [INF] from='mgr.24544 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-07-15T06:55:01.623 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:01 smithi089 bash[21793]: audit 2024-07-15T06:55:01.112244+0000 mon.b (mon.1) 54 : audit [INF] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-07-15T06:55:01.623 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:01 smithi089 bash[21793]: audit 2024-07-15T06:55:01.112705+0000 mon.a (mon.0) 984 : audit [INF] from='mgr.24544 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-07-15T06:55:01.623 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:01 smithi089 bash[31372]: audit 2024-07-15T06:55:00.512815+0000 mon.b (mon.1) 42 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-07-15T06:55:01.623 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:01 smithi089 bash[31372]: audit 2024-07-15T06:55:00.513257+0000 mon.b (mon.1) 43 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-07-15T06:55:01.623 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:01 smithi089 bash[31372]: audit 2024-07-15T06:55:00.513664+0000 mon.b (mon.1) 44 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-07-15T06:55:01.624 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:01 smithi089 bash[31372]: audit 2024-07-15T06:55:00.514179+0000 mon.b (mon.1) 45 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-07-15T06:55:01.624 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:01 smithi089 bash[31372]: audit 2024-07-15T06:55:00.514708+0000 mon.b (mon.1) 46 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-07-15T06:55:01.624 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:01 smithi089 bash[31372]: audit 2024-07-15T06:55:00.515238+0000 mon.b (mon.1) 47 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-07-15T06:55:01.624 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:01 smithi089 bash[31372]: audit 2024-07-15T06:55:00.516340+0000 mon.b (mon.1) 48 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-07-15T06:55:01.624 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:01 smithi089 bash[31372]: audit 2024-07-15T06:55:00.516885+0000 mon.b (mon.1) 49 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-07-15T06:55:01.624 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:01 smithi089 bash[31372]: audit 2024-07-15T06:55:00.517600+0000 mon.b (mon.1) 50 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-07-15T06:55:01.624 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:01 smithi089 bash[31372]: audit 2024-07-15T06:55:00.891838+0000 mon.a (mon.0) 981 : audit [DBG] from='client.? 172.21.15.89:0/4199570889' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:55:01.624 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:01 smithi089 bash[31372]: cluster 2024-07-15T06:55:01.025511+0000 mon.a (mon.0) 982 : cluster [INF] Manager daemon x is now available 2024-07-15T06:55:01.624 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:01 smithi089 bash[31372]: audit 2024-07-15T06:55:01.053755+0000 mon.b (mon.1) 51 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:55:01.624 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:01 smithi089 bash[31372]: audit 2024-07-15T06:55:01.080090+0000 mon.b (mon.1) 52 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:55:01.624 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:01 smithi089 bash[31372]: audit 2024-07-15T06:55:01.097903+0000 mon.b (mon.1) 53 : audit [INF] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-07-15T06:55:01.624 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:01 smithi089 bash[31372]: audit 2024-07-15T06:55:01.098559+0000 mon.a (mon.0) 983 : audit [INF] from='mgr.24544 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-07-15T06:55:01.624 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:01 smithi089 bash[31372]: audit 2024-07-15T06:55:01.112244+0000 mon.b (mon.1) 54 : audit [INF] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-07-15T06:55:01.624 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:01 smithi089 bash[31372]: audit 2024-07-15T06:55:01.112705+0000 mon.a (mon.0) 984 : audit [INF] from='mgr.24544 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-07-15T06:55:02.116 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:55:01 smithi089 bash[22045]: debug 2024-07-15T06:55:01.903+0000 7f30c34eeec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-07-15T06:55:02.116 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:55:01 smithi089 bash[22045]: debug 2024-07-15T06:55:01.967+0000 7f30c34eeec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-07-15T06:55:02.369 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:55:02 smithi089 bash[22045]: debug 2024-07-15T06:55:02.119+0000 7f30c34eeec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-07-15T06:55:02.869 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:55:02 smithi089 bash[22045]: debug 2024-07-15T06:55:02.770+0000 7f30c34eeec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-07-15T06:55:02.869 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:55:02 smithi089 bash[22045]: debug 2024-07-15T06:55:02.842+0000 7f30c34eeec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-07-15T06:55:02.869 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:02 smithi089 bash[31372]: cephadm 2024-07-15T06:55:01.582569+0000 mgr.x (mgr.24544) 1 : cephadm [INF] Deploying cephadm binary to smithi134 2024-07-15T06:55:02.869 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:02 smithi089 bash[31372]: cephadm 2024-07-15T06:55:01.583815+0000 mgr.x (mgr.24544) 2 : cephadm [INF] Deploying cephadm binary to smithi089 2024-07-15T06:55:02.869 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:02 smithi089 bash[31372]: cluster 2024-07-15T06:55:01.592158+0000 mon.a (mon.0) 985 : cluster [DBG] mgrmap e23: x(active, since 1.09891s) 2024-07-15T06:55:02.869 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:02 smithi089 bash[31372]: cluster 2024-07-15T06:55:01.629836+0000 mgr.x (mgr.24544) 3 : cluster [DBG] pgmap v3: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:55:02.869 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:02 smithi089 bash[31372]: audit 2024-07-15T06:55:01.744380+0000 mon.a (mon.0) 986 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:02.870 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:02 smithi089 bash[31372]: audit 2024-07-15T06:55:01.769596+0000 mon.a (mon.0) 987 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:02.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:02 smithi089 bash[21793]: cephadm 2024-07-15T06:55:01.582569+0000 mgr.x (mgr.24544) 1 : cephadm [INF] Deploying cephadm binary to smithi134 2024-07-15T06:55:02.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:02 smithi089 bash[21793]: cephadm 2024-07-15T06:55:01.583815+0000 mgr.x (mgr.24544) 2 : cephadm [INF] Deploying cephadm binary to smithi089 2024-07-15T06:55:02.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:02 smithi089 bash[21793]: cluster 2024-07-15T06:55:01.592158+0000 mon.a (mon.0) 985 : cluster [DBG] mgrmap e23: x(active, since 1.09891s) 2024-07-15T06:55:02.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:02 smithi089 bash[21793]: cluster 2024-07-15T06:55:01.629836+0000 mgr.x (mgr.24544) 3 : cluster [DBG] pgmap v3: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:55:02.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:02 smithi089 bash[21793]: audit 2024-07-15T06:55:01.744380+0000 mon.a (mon.0) 986 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:02.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:02 smithi089 bash[21793]: audit 2024-07-15T06:55:01.769596+0000 mon.a (mon.0) 987 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:03.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:02 smithi134 bash[24068]: cephadm 2024-07-15T06:55:01.582569+0000 mgr.x (mgr.24544) 1 : cephadm [INF] Deploying cephadm binary to smithi134 2024-07-15T06:55:03.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:02 smithi134 bash[24068]: cephadm 2024-07-15T06:55:01.583815+0000 mgr.x (mgr.24544) 2 : cephadm [INF] Deploying cephadm binary to smithi089 2024-07-15T06:55:03.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:02 smithi134 bash[24068]: cluster 2024-07-15T06:55:01.592158+0000 mon.a (mon.0) 985 : cluster [DBG] mgrmap e23: x(active, since 1.09891s) 2024-07-15T06:55:03.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:02 smithi134 bash[24068]: cluster 2024-07-15T06:55:01.629836+0000 mgr.x (mgr.24544) 3 : cluster [DBG] pgmap v3: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:55:03.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:02 smithi134 bash[24068]: audit 2024-07-15T06:55:01.744380+0000 mon.a (mon.0) 986 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:03.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:02 smithi134 bash[24068]: audit 2024-07-15T06:55:01.769596+0000 mon.a (mon.0) 987 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:03.054 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:55:02 smithi134 bash[52600]: debug 2024-07-15T06:55:02.751+0000 7faa2077c640 -1 log_channel(cephadm) log [ERR] : [15/Jul/2024:06:55:02] ENGINE Error in HTTPServer.serve 2024-07-15T06:55:03.055 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:55:02 smithi134 bash[52600]: Traceback (most recent call last): 2024-07-15T06:55:03.055 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:55:02 smithi134 bash[52600]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-07-15T06:55:03.055 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:55:02 smithi134 bash[52600]: self._connections.run(self.expiration_interval) 2024-07-15T06:55:03.055 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:55:02 smithi134 bash[52600]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-07-15T06:55:03.055 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:55:02 smithi134 bash[52600]: self._run(expiration_interval) 2024-07-15T06:55:03.055 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:55:02 smithi134 bash[52600]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-07-15T06:55:03.055 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:55:02 smithi134 bash[52600]: new_conn = self._from_server_socket(self.server.socket) 2024-07-15T06:55:03.055 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:55:02 smithi134 bash[52600]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-07-15T06:55:03.055 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:55:02 smithi134 bash[52600]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-07-15T06:55:03.055 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:55:02 smithi134 bash[52600]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-07-15T06:55:03.055 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:55:02 smithi134 bash[52600]: s = self.context.wrap_socket( 2024-07-15T06:55:03.055 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:55:02 smithi134 bash[52600]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-07-15T06:55:03.055 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:55:02 smithi134 bash[52600]: return self.sslsocket_class._create( 2024-07-15T06:55:03.055 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:55:02 smithi134 bash[52600]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-07-15T06:55:03.056 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:55:02 smithi134 bash[52600]: self.do_handshake() 2024-07-15T06:55:03.056 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:55:02 smithi134 bash[52600]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-07-15T06:55:03.056 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:55:02 smithi134 bash[52600]: self._sslobj.do_handshake() 2024-07-15T06:55:03.056 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:55:02 smithi134 bash[52600]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-07-15T06:55:03.220 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:55:02 smithi089 bash[22045]: debug 2024-07-15T06:55:02.910+0000 7f30c34eeec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-07-15T06:55:03.220 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:55:03 smithi089 bash[22045]: debug 2024-07-15T06:55:03.074+0000 7f30c34eeec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-07-15T06:55:03.221 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:55:03 smithi089 bash[22045]: debug 2024-07-15T06:55:03.154+0000 7f30c34eeec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-07-15T06:55:03.221 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:55:03 smithi089 bash[22045]: debug 2024-07-15T06:55:03.222+0000 7f30c34eeec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-07-15T06:55:03.522 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:55:03 smithi089 bash[22045]: debug 2024-07-15T06:55:03.458+0000 7f30c34eeec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-07-15T06:55:03.522 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:55:03 smithi089 bash[22045]: debug 2024-07-15T06:55:03.526+0000 7f30c34eeec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-07-15T06:55:03.777 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:55:03 smithi089 bash[22045]: debug 2024-07-15T06:55:03.702+0000 7f30c34eeec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-07-15T06:55:03.777 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:03 smithi089 bash[21793]: cluster 2024-07-15T06:55:02.516528+0000 mgr.x (mgr.24544) 4 : cluster [DBG] pgmap v4: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:55:03.777 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:03 smithi089 bash[21793]: cephadm 2024-07-15T06:55:02.531102+0000 mgr.x (mgr.24544) 5 : cephadm [INF] [15/Jul/2024:06:55:02] ENGINE Bus STARTING 2024-07-15T06:55:03.778 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:03 smithi089 bash[21793]: cephadm 2024-07-15T06:55:02.632784+0000 mgr.x (mgr.24544) 6 : cephadm [INF] [15/Jul/2024:06:55:02] ENGINE Serving on http://172.21.15.134:8765 2024-07-15T06:55:03.778 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:03 smithi089 bash[21793]: cephadm 2024-07-15T06:55:02.753797+0000 mgr.x (mgr.24544) 7 : cephadm [INF] [15/Jul/2024:06:55:02] ENGINE Serving on https://172.21.15.134:7150 2024-07-15T06:55:03.778 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:03 smithi089 bash[21793]: cephadm 2024-07-15T06:55:02.754102+0000 mgr.x (mgr.24544) 8 : cephadm [INF] [15/Jul/2024:06:55:02] ENGINE Bus STARTED 2024-07-15T06:55:03.778 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:03 smithi089 bash[21793]: cephadm 2024-07-15T06:55:02.755707+0000 mgr.x (mgr.24544) 9 : cephadm [ERR] [15/Jul/2024:06:55:02] ENGINE Error in HTTPServer.serve 2024-07-15T06:55:03.778 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:03 smithi089 bash[21793]: Traceback (most recent call last): 2024-07-15T06:55:03.778 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:03 smithi089 bash[21793]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-07-15T06:55:03.778 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:03 smithi089 bash[21793]: self._connections.run(self.expiration_interval) 2024-07-15T06:55:03.778 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:03 smithi089 bash[21793]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-07-15T06:55:03.778 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:03 smithi089 bash[21793]: self._run(expiration_interval) 2024-07-15T06:55:03.778 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:03 smithi089 bash[21793]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-07-15T06:55:03.778 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:03 smithi089 bash[21793]: new_conn = self._from_server_socket(self.server.socket) 2024-07-15T06:55:03.778 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:03 smithi089 bash[21793]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-07-15T06:55:03.778 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:03 smithi089 bash[21793]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-07-15T06:55:03.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:03 smithi089 bash[21793]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-07-15T06:55:03.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:03 smithi089 bash[21793]: s = self.context.wrap_socket( 2024-07-15T06:55:03.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:03 smithi089 bash[21793]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-07-15T06:55:03.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:03 smithi089 bash[21793]: return self.sslsocket_class._create( 2024-07-15T06:55:03.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:03 smithi089 bash[21793]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-07-15T06:55:03.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:03 smithi089 bash[21793]: self.do_handshake() 2024-07-15T06:55:03.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:03 smithi089 bash[21793]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-07-15T06:55:03.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:03 smithi089 bash[21793]: self._sslobj.do_handshake() 2024-07-15T06:55:03.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:03 smithi089 bash[21793]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-07-15T06:55:03.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:03 smithi089 bash[21793]: audit 2024-07-15T06:55:03.245297+0000 mon.a (mon.0) 988 : audit [DBG] from='client.? 172.21.15.89:0/3894301268' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:55:03.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:03 smithi089 bash[31372]: cluster 2024-07-15T06:55:02.516528+0000 mgr.x (mgr.24544) 4 : cluster [DBG] pgmap v4: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:55:03.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:03 smithi089 bash[31372]: cephadm 2024-07-15T06:55:02.531102+0000 mgr.x (mgr.24544) 5 : cephadm [INF] [15/Jul/2024:06:55:02] ENGINE Bus STARTING 2024-07-15T06:55:03.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:03 smithi089 bash[31372]: cephadm 2024-07-15T06:55:02.632784+0000 mgr.x (mgr.24544) 6 : cephadm [INF] [15/Jul/2024:06:55:02] ENGINE Serving on http://172.21.15.134:8765 2024-07-15T06:55:03.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:03 smithi089 bash[31372]: cephadm 2024-07-15T06:55:02.753797+0000 mgr.x (mgr.24544) 7 : cephadm [INF] [15/Jul/2024:06:55:02] ENGINE Serving on https://172.21.15.134:7150 2024-07-15T06:55:03.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:03 smithi089 bash[31372]: cephadm 2024-07-15T06:55:02.754102+0000 mgr.x (mgr.24544) 8 : cephadm [INF] [15/Jul/2024:06:55:02] ENGINE Bus STARTED 2024-07-15T06:55:03.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:03 smithi089 bash[31372]: cephadm 2024-07-15T06:55:02.755707+0000 mgr.x (mgr.24544) 9 : cephadm [ERR] [15/Jul/2024:06:55:02] ENGINE Error in HTTPServer.serve 2024-07-15T06:55:03.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:03 smithi089 bash[31372]: Traceback (most recent call last): 2024-07-15T06:55:03.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:03 smithi089 bash[31372]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-07-15T06:55:03.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:03 smithi089 bash[31372]: self._connections.run(self.expiration_interval) 2024-07-15T06:55:03.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:03 smithi089 bash[31372]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-07-15T06:55:03.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:03 smithi089 bash[31372]: self._run(expiration_interval) 2024-07-15T06:55:03.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:03 smithi089 bash[31372]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-07-15T06:55:03.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:03 smithi089 bash[31372]: new_conn = self._from_server_socket(self.server.socket) 2024-07-15T06:55:03.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:03 smithi089 bash[31372]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-07-15T06:55:03.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:03 smithi089 bash[31372]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-07-15T06:55:03.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:03 smithi089 bash[31372]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-07-15T06:55:03.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:03 smithi089 bash[31372]: s = self.context.wrap_socket( 2024-07-15T06:55:03.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:03 smithi089 bash[31372]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-07-15T06:55:03.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:03 smithi089 bash[31372]: return self.sslsocket_class._create( 2024-07-15T06:55:03.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:03 smithi089 bash[31372]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-07-15T06:55:03.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:03 smithi089 bash[31372]: self.do_handshake() 2024-07-15T06:55:03.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:03 smithi089 bash[31372]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-07-15T06:55:03.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:03 smithi089 bash[31372]: self._sslobj.do_handshake() 2024-07-15T06:55:03.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:03 smithi089 bash[31372]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-07-15T06:55:03.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:03 smithi089 bash[31372]: audit 2024-07-15T06:55:03.245297+0000 mon.a (mon.0) 988 : audit [DBG] from='client.? 172.21.15.89:0/3894301268' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:55:04.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:03 smithi134 bash[24068]: cluster 2024-07-15T06:55:02.516528+0000 mgr.x (mgr.24544) 4 : cluster [DBG] pgmap v4: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:55:04.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:03 smithi134 bash[24068]: cephadm 2024-07-15T06:55:02.531102+0000 mgr.x (mgr.24544) 5 : cephadm [INF] [15/Jul/2024:06:55:02] ENGINE Bus STARTING 2024-07-15T06:55:04.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:03 smithi134 bash[24068]: cephadm 2024-07-15T06:55:02.632784+0000 mgr.x (mgr.24544) 6 : cephadm [INF] [15/Jul/2024:06:55:02] ENGINE Serving on http://172.21.15.134:8765 2024-07-15T06:55:04.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:03 smithi134 bash[24068]: cephadm 2024-07-15T06:55:02.753797+0000 mgr.x (mgr.24544) 7 : cephadm [INF] [15/Jul/2024:06:55:02] ENGINE Serving on https://172.21.15.134:7150 2024-07-15T06:55:04.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:03 smithi134 bash[24068]: cephadm 2024-07-15T06:55:02.754102+0000 mgr.x (mgr.24544) 8 : cephadm [INF] [15/Jul/2024:06:55:02] ENGINE Bus STARTED 2024-07-15T06:55:04.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:03 smithi134 bash[24068]: cephadm 2024-07-15T06:55:02.755707+0000 mgr.x (mgr.24544) 9 : cephadm [ERR] [15/Jul/2024:06:55:02] ENGINE Error in HTTPServer.serve 2024-07-15T06:55:04.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:03 smithi134 bash[24068]: Traceback (most recent call last): 2024-07-15T06:55:04.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:03 smithi134 bash[24068]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-07-15T06:55:04.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:03 smithi134 bash[24068]: self._connections.run(self.expiration_interval) 2024-07-15T06:55:04.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:03 smithi134 bash[24068]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-07-15T06:55:04.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:03 smithi134 bash[24068]: self._run(expiration_interval) 2024-07-15T06:55:04.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:03 smithi134 bash[24068]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-07-15T06:55:04.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:03 smithi134 bash[24068]: new_conn = self._from_server_socket(self.server.socket) 2024-07-15T06:55:04.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:03 smithi134 bash[24068]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-07-15T06:55:04.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:03 smithi134 bash[24068]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-07-15T06:55:04.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:03 smithi134 bash[24068]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-07-15T06:55:04.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:03 smithi134 bash[24068]: s = self.context.wrap_socket( 2024-07-15T06:55:04.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:03 smithi134 bash[24068]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-07-15T06:55:04.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:03 smithi134 bash[24068]: return self.sslsocket_class._create( 2024-07-15T06:55:04.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:03 smithi134 bash[24068]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-07-15T06:55:04.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:03 smithi134 bash[24068]: self.do_handshake() 2024-07-15T06:55:04.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:03 smithi134 bash[24068]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-07-15T06:55:04.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:03 smithi134 bash[24068]: self._sslobj.do_handshake() 2024-07-15T06:55:04.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:03 smithi134 bash[24068]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-07-15T06:55:04.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:03 smithi134 bash[24068]: audit 2024-07-15T06:55:03.245297+0000 mon.a (mon.0) 988 : audit [DBG] from='client.? 172.21.15.89:0/3894301268' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:55:04.118 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:55:03 smithi089 bash[22045]: debug 2024-07-15T06:55:03.778+0000 7f30c34eeec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-07-15T06:55:04.119 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:55:04 smithi089 bash[22045]: debug 2024-07-15T06:55:04.014+0000 7f30c34eeec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-07-15T06:55:04.614 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:55:04 smithi089 bash[22045]: debug 2024-07-15T06:55:04.178+0000 7f30c34eeec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-07-15T06:55:04.614 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:55:04 smithi089 bash[22045]: debug 2024-07-15T06:55:04.254+0000 7f30c34eeec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-07-15T06:55:04.614 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:55:04 smithi089 bash[22045]: debug 2024-07-15T06:55:04.410+0000 7f30c34eeec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-07-15T06:55:04.869 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:04 smithi089 bash[31372]: cluster 2024-07-15T06:55:03.608554+0000 mon.a (mon.0) 989 : cluster [DBG] mgrmap e24: x(active, since 3s) 2024-07-15T06:55:04.869 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:04 smithi089 bash[21793]: cluster 2024-07-15T06:55:03.608554+0000 mon.a (mon.0) 989 : cluster [DBG] mgrmap e24: x(active, since 3s) 2024-07-15T06:55:05.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:04 smithi134 bash[24068]: cluster 2024-07-15T06:55:03.608554+0000 mon.a (mon.0) 989 : cluster [DBG] mgrmap e24: x(active, since 3s) 2024-07-15T06:55:05.489 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:55:05 smithi089 bash[22045]: debug 2024-07-15T06:55:05.210+0000 7f30c34eeec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-07-15T06:55:05.489 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:55:05 smithi089 bash[22045]: debug 2024-07-15T06:55:05.286+0000 7f30c34eeec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-07-15T06:55:05.869 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:05 smithi089 bash[31372]: cluster 2024-07-15T06:55:04.517176+0000 mgr.x (mgr.24544) 10 : cluster [DBG] pgmap v5: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:55:05.869 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:05 smithi089 bash[31372]: audit 2024-07-15T06:55:05.602806+0000 mon.a (mon.0) 990 : audit [DBG] from='client.? 172.21.15.89:0/1338059183' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:55:05.869 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:55:05 smithi089 bash[22045]: debug 2024-07-15T06:55:05.490+0000 7f30c34eeec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-07-15T06:55:05.869 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:55:05 smithi089 bash[22045]: debug 2024-07-15T06:55:05.566+0000 7f30c34eeec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-07-15T06:55:05.869 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:05 smithi089 bash[21793]: cluster 2024-07-15T06:55:04.517176+0000 mgr.x (mgr.24544) 10 : cluster [DBG] pgmap v5: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:55:05.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:05 smithi089 bash[21793]: audit 2024-07-15T06:55:05.602806+0000 mon.a (mon.0) 990 : audit [DBG] from='client.? 172.21.15.89:0/1338059183' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:55:06.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:05 smithi134 bash[24068]: cluster 2024-07-15T06:55:04.517176+0000 mgr.x (mgr.24544) 10 : cluster [DBG] pgmap v5: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:55:06.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:05 smithi134 bash[24068]: audit 2024-07-15T06:55:05.602806+0000 mon.a (mon.0) 990 : audit [DBG] from='client.? 172.21.15.89:0/1338059183' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:55:06.369 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:55:05 smithi089 bash[22045]: debug 2024-07-15T06:55:05.890+0000 7f30c34eeec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-07-15T06:55:06.369 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:55:05 smithi089 bash[22045]: debug 2024-07-15T06:55:05.970+0000 7f30c34eeec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-07-15T06:55:06.369 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:55:05 smithi089 bash[22045]: [15/Jul/2024:06:55:05] ENGINE Bus STARTING 2024-07-15T06:55:06.369 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:55:05 smithi089 bash[22045]: CherryPy Checker: 2024-07-15T06:55:06.369 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:55:05 smithi089 bash[22045]: The Application mounted at '' has an empty config. 2024-07-15T06:55:06.369 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:55:06 smithi089 bash[22045]: [15/Jul/2024:06:55:06] ENGINE Serving on http://:::9283 2024-07-15T06:55:06.369 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:55:06 smithi089 bash[22045]: [15/Jul/2024:06:55:06] ENGINE Bus STARTED 2024-07-15T06:55:06.963 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:06 smithi134 bash[24068]: cluster 2024-07-15T06:55:05.632208+0000 mon.a (mon.0) 991 : cluster [DBG] mgrmap e25: x(active, since 5s) 2024-07-15T06:55:06.963 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:06 smithi134 bash[24068]: cluster 2024-07-15T06:55:05.993252+0000 mon.a (mon.0) 992 : cluster [DBG] Standby manager daemon y started 2024-07-15T06:55:06.963 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:06 smithi134 bash[24068]: audit 2024-07-15T06:55:05.997170+0000 mon.a (mon.0) 993 : audit [DBG] from='mgr.? 172.21.15.89:0/1084255045' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-07-15T06:55:06.963 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:06 smithi134 bash[24068]: audit 2024-07-15T06:55:05.999449+0000 mon.a (mon.0) 994 : audit [DBG] from='mgr.? 172.21.15.89:0/1084255045' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-07-15T06:55:06.963 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:06 smithi134 bash[24068]: audit 2024-07-15T06:55:06.001550+0000 mon.a (mon.0) 995 : audit [DBG] from='mgr.? 172.21.15.89:0/1084255045' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-07-15T06:55:06.963 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:06 smithi134 bash[24068]: audit 2024-07-15T06:55:06.001907+0000 mon.a (mon.0) 996 : audit [DBG] from='mgr.? 172.21.15.89:0/1084255045' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-07-15T06:55:07.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:06 smithi089 bash[21793]: cluster 2024-07-15T06:55:05.632208+0000 mon.a (mon.0) 991 : cluster [DBG] mgrmap e25: x(active, since 5s) 2024-07-15T06:55:07.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:06 smithi089 bash[21793]: cluster 2024-07-15T06:55:05.993252+0000 mon.a (mon.0) 992 : cluster [DBG] Standby manager daemon y started 2024-07-15T06:55:07.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:06 smithi089 bash[21793]: audit 2024-07-15T06:55:05.997170+0000 mon.a (mon.0) 993 : audit [DBG] from='mgr.? 172.21.15.89:0/1084255045' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-07-15T06:55:07.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:06 smithi089 bash[21793]: audit 2024-07-15T06:55:05.999449+0000 mon.a (mon.0) 994 : audit [DBG] from='mgr.? 172.21.15.89:0/1084255045' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-07-15T06:55:07.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:06 smithi089 bash[21793]: audit 2024-07-15T06:55:06.001550+0000 mon.a (mon.0) 995 : audit [DBG] from='mgr.? 172.21.15.89:0/1084255045' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-07-15T06:55:07.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:06 smithi089 bash[21793]: audit 2024-07-15T06:55:06.001907+0000 mon.a (mon.0) 996 : audit [DBG] from='mgr.? 172.21.15.89:0/1084255045' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-07-15T06:55:07.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:06 smithi089 bash[31372]: cluster 2024-07-15T06:55:05.632208+0000 mon.a (mon.0) 991 : cluster [DBG] mgrmap e25: x(active, since 5s) 2024-07-15T06:55:07.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:06 smithi089 bash[31372]: cluster 2024-07-15T06:55:05.993252+0000 mon.a (mon.0) 992 : cluster [DBG] Standby manager daemon y started 2024-07-15T06:55:07.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:06 smithi089 bash[31372]: audit 2024-07-15T06:55:05.997170+0000 mon.a (mon.0) 993 : audit [DBG] from='mgr.? 172.21.15.89:0/1084255045' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-07-15T06:55:07.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:06 smithi089 bash[31372]: audit 2024-07-15T06:55:05.999449+0000 mon.a (mon.0) 994 : audit [DBG] from='mgr.? 172.21.15.89:0/1084255045' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-07-15T06:55:07.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:06 smithi089 bash[31372]: audit 2024-07-15T06:55:06.001550+0000 mon.a (mon.0) 995 : audit [DBG] from='mgr.? 172.21.15.89:0/1084255045' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-07-15T06:55:07.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:06 smithi089 bash[31372]: audit 2024-07-15T06:55:06.001907+0000 mon.a (mon.0) 996 : audit [DBG] from='mgr.? 172.21.15.89:0/1084255045' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-07-15T06:55:07.304 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:55:06 smithi134 bash[52600]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:55:06] "GET /metrics HTTP/1.1" 200 30960 "" "Prometheus/2.43.0" 2024-07-15T06:55:08.053 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:07 smithi134 bash[24068]: cluster 2024-07-15T06:55:06.517843+0000 mgr.x (mgr.24544) 11 : cluster [DBG] pgmap v6: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:55:08.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:07 smithi134 bash[24068]: cluster 2024-07-15T06:55:06.643987+0000 mon.a (mon.0) 997 : cluster [DBG] mgrmap e26: x(active, since 6s), standbys: y 2024-07-15T06:55:08.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:07 smithi134 bash[24068]: audit 2024-07-15T06:55:06.647188+0000 mon.b (mon.1) 55 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-07-15T06:55:08.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:07 smithi089 bash[31372]: cluster 2024-07-15T06:55:06.517843+0000 mgr.x (mgr.24544) 11 : cluster [DBG] pgmap v6: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:55:08.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:07 smithi089 bash[31372]: cluster 2024-07-15T06:55:06.643987+0000 mon.a (mon.0) 997 : cluster [DBG] mgrmap e26: x(active, since 6s), standbys: y 2024-07-15T06:55:08.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:07 smithi089 bash[31372]: audit 2024-07-15T06:55:06.647188+0000 mon.b (mon.1) 55 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-07-15T06:55:08.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:07 smithi089 bash[21793]: cluster 2024-07-15T06:55:06.517843+0000 mgr.x (mgr.24544) 11 : cluster [DBG] pgmap v6: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:55:08.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:07 smithi089 bash[21793]: cluster 2024-07-15T06:55:06.643987+0000 mon.a (mon.0) 997 : cluster [DBG] mgrmap e26: x(active, since 6s), standbys: y 2024-07-15T06:55:08.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:07 smithi089 bash[21793]: audit 2024-07-15T06:55:06.647188+0000 mon.b (mon.1) 55 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-07-15T06:55:09.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:08 smithi134 bash[24068]: audit 2024-07-15T06:55:07.667565+0000 mon.a (mon.0) 998 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:09.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:08 smithi134 bash[24068]: audit 2024-07-15T06:55:07.675415+0000 mon.a (mon.0) 999 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:09.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:08 smithi134 bash[24068]: audit 2024-07-15T06:55:07.944848+0000 mon.a (mon.0) 1000 : audit [DBG] from='client.? 172.21.15.89:0/2678124908' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:55:09.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:08 smithi134 bash[24068]: audit 2024-07-15T06:55:08.140418+0000 mon.a (mon.0) 1001 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:09.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:08 smithi134 bash[24068]: audit 2024-07-15T06:55:08.149309+0000 mon.a (mon.0) 1002 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:09.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:08 smithi134 bash[24068]: audit 2024-07-15T06:55:08.403364+0000 mon.a (mon.0) 1003 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:09.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:08 smithi134 bash[24068]: audit 2024-07-15T06:55:08.410592+0000 mon.a (mon.0) 1004 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:09.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:08 smithi134 bash[24068]: audit 2024-07-15T06:55:08.412706+0000 mon.b (mon.1) 56 : audit [INF] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:55:09.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:08 smithi134 bash[24068]: audit 2024-07-15T06:55:08.412893+0000 mon.a (mon.0) 1005 : audit [INF] from='mgr.24544 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:55:09.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:08 smithi134 bash[24068]: audit 2024-07-15T06:55:08.414151+0000 mon.b (mon.1) 57 : audit [INF] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:55:09.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:08 smithi134 bash[24068]: audit 2024-07-15T06:55:08.414253+0000 mon.a (mon.0) 1006 : audit [INF] from='mgr.24544 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:55:09.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:08 smithi134 bash[24068]: audit 2024-07-15T06:55:08.415310+0000 mon.b (mon.1) 58 : audit [INF] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:55:09.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:08 smithi134 bash[24068]: audit 2024-07-15T06:55:08.415431+0000 mon.a (mon.0) 1007 : audit [INF] from='mgr.24544 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:55:09.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:08 smithi134 bash[24068]: audit 2024-07-15T06:55:08.416479+0000 mon.b (mon.1) 59 : audit [INF] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:55:09.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:08 smithi134 bash[24068]: audit 2024-07-15T06:55:08.416561+0000 mon.a (mon.0) 1008 : audit [INF] from='mgr.24544 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:55:09.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:08 smithi134 bash[24068]: audit 2024-07-15T06:55:08.421373+0000 mon.a (mon.0) 1009 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:09.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:08 smithi089 bash[21793]: audit 2024-07-15T06:55:07.667565+0000 mon.a (mon.0) 998 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:09.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:08 smithi089 bash[21793]: audit 2024-07-15T06:55:07.675415+0000 mon.a (mon.0) 999 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:09.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:08 smithi089 bash[21793]: audit 2024-07-15T06:55:07.944848+0000 mon.a (mon.0) 1000 : audit [DBG] from='client.? 172.21.15.89:0/2678124908' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:55:09.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:08 smithi089 bash[21793]: audit 2024-07-15T06:55:08.140418+0000 mon.a (mon.0) 1001 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:09.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:08 smithi089 bash[21793]: audit 2024-07-15T06:55:08.149309+0000 mon.a (mon.0) 1002 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:09.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:08 smithi089 bash[21793]: audit 2024-07-15T06:55:08.403364+0000 mon.a (mon.0) 1003 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:09.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:08 smithi089 bash[21793]: audit 2024-07-15T06:55:08.410592+0000 mon.a (mon.0) 1004 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:09.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:08 smithi089 bash[21793]: audit 2024-07-15T06:55:08.412706+0000 mon.b (mon.1) 56 : audit [INF] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:55:09.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:08 smithi089 bash[21793]: audit 2024-07-15T06:55:08.412893+0000 mon.a (mon.0) 1005 : audit [INF] from='mgr.24544 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:55:09.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:08 smithi089 bash[21793]: audit 2024-07-15T06:55:08.414151+0000 mon.b (mon.1) 57 : audit [INF] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:55:09.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:08 smithi089 bash[21793]: audit 2024-07-15T06:55:08.414253+0000 mon.a (mon.0) 1006 : audit [INF] from='mgr.24544 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:55:09.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:08 smithi089 bash[21793]: audit 2024-07-15T06:55:08.415310+0000 mon.b (mon.1) 58 : audit [INF] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:55:09.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:08 smithi089 bash[21793]: audit 2024-07-15T06:55:08.415431+0000 mon.a (mon.0) 1007 : audit [INF] from='mgr.24544 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:55:09.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:08 smithi089 bash[21793]: audit 2024-07-15T06:55:08.416479+0000 mon.b (mon.1) 59 : audit [INF] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:55:09.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:08 smithi089 bash[21793]: audit 2024-07-15T06:55:08.416561+0000 mon.a (mon.0) 1008 : audit [INF] from='mgr.24544 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:55:09.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:08 smithi089 bash[21793]: audit 2024-07-15T06:55:08.421373+0000 mon.a (mon.0) 1009 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:09.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:08 smithi089 bash[31372]: audit 2024-07-15T06:55:07.667565+0000 mon.a (mon.0) 998 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:09.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:08 smithi089 bash[31372]: audit 2024-07-15T06:55:07.675415+0000 mon.a (mon.0) 999 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:09.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:08 smithi089 bash[31372]: audit 2024-07-15T06:55:07.944848+0000 mon.a (mon.0) 1000 : audit [DBG] from='client.? 172.21.15.89:0/2678124908' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:55:09.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:08 smithi089 bash[31372]: audit 2024-07-15T06:55:08.140418+0000 mon.a (mon.0) 1001 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:09.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:08 smithi089 bash[31372]: audit 2024-07-15T06:55:08.149309+0000 mon.a (mon.0) 1002 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:09.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:08 smithi089 bash[31372]: audit 2024-07-15T06:55:08.403364+0000 mon.a (mon.0) 1003 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:09.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:08 smithi089 bash[31372]: audit 2024-07-15T06:55:08.410592+0000 mon.a (mon.0) 1004 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:09.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:08 smithi089 bash[31372]: audit 2024-07-15T06:55:08.412706+0000 mon.b (mon.1) 56 : audit [INF] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:55:09.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:08 smithi089 bash[31372]: audit 2024-07-15T06:55:08.412893+0000 mon.a (mon.0) 1005 : audit [INF] from='mgr.24544 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:55:09.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:08 smithi089 bash[31372]: audit 2024-07-15T06:55:08.414151+0000 mon.b (mon.1) 57 : audit [INF] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:55:09.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:08 smithi089 bash[31372]: audit 2024-07-15T06:55:08.414253+0000 mon.a (mon.0) 1006 : audit [INF] from='mgr.24544 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:55:09.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:08 smithi089 bash[31372]: audit 2024-07-15T06:55:08.415310+0000 mon.b (mon.1) 58 : audit [INF] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:55:09.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:08 smithi089 bash[31372]: audit 2024-07-15T06:55:08.415431+0000 mon.a (mon.0) 1007 : audit [INF] from='mgr.24544 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:55:09.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:08 smithi089 bash[31372]: audit 2024-07-15T06:55:08.416479+0000 mon.b (mon.1) 59 : audit [INF] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:55:09.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:08 smithi089 bash[31372]: audit 2024-07-15T06:55:08.416561+0000 mon.a (mon.0) 1008 : audit [INF] from='mgr.24544 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:55:09.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:08 smithi089 bash[31372]: audit 2024-07-15T06:55:08.421373+0000 mon.a (mon.0) 1009 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:10.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:09 smithi134 bash[24068]: cephadm 2024-07-15T06:55:08.417469+0000 mgr.x (mgr.24544) 12 : cephadm [INF] Adjusting osd_memory_target on smithi089 to 2524M 2024-07-15T06:55:10.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:09 smithi134 bash[24068]: cluster 2024-07-15T06:55:08.518416+0000 mgr.x (mgr.24544) 13 : cluster [DBG] pgmap v7: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 16 op/s 2024-07-15T06:55:10.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:09 smithi134 bash[24068]: audit 2024-07-15T06:55:08.894319+0000 mon.a (mon.0) 1010 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:10.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:09 smithi134 bash[24068]: audit 2024-07-15T06:55:08.905343+0000 mon.a (mon.0) 1011 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:10.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:09 smithi089 bash[21793]: cephadm 2024-07-15T06:55:08.417469+0000 mgr.x (mgr.24544) 12 : cephadm [INF] Adjusting osd_memory_target on smithi089 to 2524M 2024-07-15T06:55:10.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:09 smithi089 bash[21793]: cluster 2024-07-15T06:55:08.518416+0000 mgr.x (mgr.24544) 13 : cluster [DBG] pgmap v7: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 16 op/s 2024-07-15T06:55:10.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:09 smithi089 bash[21793]: audit 2024-07-15T06:55:08.894319+0000 mon.a (mon.0) 1010 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:10.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:09 smithi089 bash[21793]: audit 2024-07-15T06:55:08.905343+0000 mon.a (mon.0) 1011 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:10.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:09 smithi089 bash[31372]: cephadm 2024-07-15T06:55:08.417469+0000 mgr.x (mgr.24544) 12 : cephadm [INF] Adjusting osd_memory_target on smithi089 to 2524M 2024-07-15T06:55:10.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:09 smithi089 bash[31372]: cluster 2024-07-15T06:55:08.518416+0000 mgr.x (mgr.24544) 13 : cluster [DBG] pgmap v7: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 16 op/s 2024-07-15T06:55:10.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:09 smithi089 bash[31372]: audit 2024-07-15T06:55:08.894319+0000 mon.a (mon.0) 1010 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:10.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:09 smithi089 bash[31372]: audit 2024-07-15T06:55:08.905343+0000 mon.a (mon.0) 1011 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:11.053 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:10 smithi134 bash[24068]: audit 2024-07-15T06:55:10.313279+0000 mon.c (mon.2) 43 : audit [DBG] from='client.? 172.21.15.89:0/3453344855' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:55:11.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:10 smithi089 bash[21793]: audit 2024-07-15T06:55:10.313279+0000 mon.c (mon.2) 43 : audit [DBG] from='client.? 172.21.15.89:0/3453344855' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:55:11.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:10 smithi089 bash[31372]: audit 2024-07-15T06:55:10.313279+0000 mon.c (mon.2) 43 : audit [DBG] from='client.? 172.21.15.89:0/3453344855' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:55:12.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:11 smithi134 bash[24068]: cluster 2024-07-15T06:55:10.519049+0000 mgr.x (mgr.24544) 14 : cluster [DBG] pgmap v8: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-07-15T06:55:12.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:11 smithi089 bash[21793]: cluster 2024-07-15T06:55:10.519049+0000 mgr.x (mgr.24544) 14 : cluster [DBG] pgmap v8: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-07-15T06:55:12.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:11 smithi089 bash[31372]: cluster 2024-07-15T06:55:10.519049+0000 mgr.x (mgr.24544) 14 : cluster [DBG] pgmap v8: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-07-15T06:55:13.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:12 smithi134 bash[24068]: audit 2024-07-15T06:55:12.676742+0000 mon.c (mon.2) 44 : audit [DBG] from='client.? 172.21.15.89:0/3398679598' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:55:13.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:12 smithi089 bash[21793]: audit 2024-07-15T06:55:12.676742+0000 mon.c (mon.2) 44 : audit [DBG] from='client.? 172.21.15.89:0/3398679598' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:55:13.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:12 smithi089 bash[31372]: audit 2024-07-15T06:55:12.676742+0000 mon.c (mon.2) 44 : audit [DBG] from='client.? 172.21.15.89:0/3398679598' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:55:13.120 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:55:12 smithi089 bash[22045]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:55:12] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-07-15T06:55:14.053 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:13 smithi134 bash[24068]: cluster 2024-07-15T06:55:12.520193+0000 mgr.x (mgr.24544) 15 : cluster [DBG] pgmap v9: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-07-15T06:55:14.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:13 smithi089 bash[21793]: cluster 2024-07-15T06:55:12.520193+0000 mgr.x (mgr.24544) 15 : cluster [DBG] pgmap v9: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-07-15T06:55:14.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:13 smithi089 bash[31372]: cluster 2024-07-15T06:55:12.520193+0000 mgr.x (mgr.24544) 15 : cluster [DBG] pgmap v9: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-07-15T06:55:16.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:15 smithi134 bash[24068]: cluster 2024-07-15T06:55:14.520894+0000 mgr.x (mgr.24544) 16 : cluster [DBG] pgmap v10: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-07-15T06:55:16.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:15 smithi134 bash[24068]: audit 2024-07-15T06:55:15.045379+0000 mon.a (mon.0) 1012 : audit [DBG] from='client.? 172.21.15.89:0/3478990709' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:55:16.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:15 smithi089 bash[31372]: cluster 2024-07-15T06:55:14.520894+0000 mgr.x (mgr.24544) 16 : cluster [DBG] pgmap v10: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-07-15T06:55:16.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:15 smithi089 bash[31372]: audit 2024-07-15T06:55:15.045379+0000 mon.a (mon.0) 1012 : audit [DBG] from='client.? 172.21.15.89:0/3478990709' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:55:16.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:15 smithi089 bash[21793]: cluster 2024-07-15T06:55:14.520894+0000 mgr.x (mgr.24544) 16 : cluster [DBG] pgmap v10: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-07-15T06:55:16.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:15 smithi089 bash[21793]: audit 2024-07-15T06:55:15.045379+0000 mon.a (mon.0) 1012 : audit [DBG] from='client.? 172.21.15.89:0/3478990709' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:55:16.962 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:16 smithi134 bash[24068]: audit 2024-07-15T06:55:16.079696+0000 mon.b (mon.1) 60 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:55:17.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:16 smithi089 bash[21793]: audit 2024-07-15T06:55:16.079696+0000 mon.b (mon.1) 60 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:55:17.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:16 smithi089 bash[31372]: audit 2024-07-15T06:55:16.079696+0000 mon.b (mon.1) 60 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:55:17.305 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:55:16 smithi134 bash[52600]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:55:16] "GET /metrics HTTP/1.1" 200 33807 "" "Prometheus/2.43.0" 2024-07-15T06:55:18.053 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:17 smithi134 bash[24068]: cluster 2024-07-15T06:55:16.521574+0000 mgr.x (mgr.24544) 17 : cluster [DBG] pgmap v11: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-07-15T06:55:18.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:17 smithi134 bash[24068]: audit 2024-07-15T06:55:17.416423+0000 mon.a (mon.0) 1013 : audit [DBG] from='client.? 172.21.15.89:0/2236019436' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:55:18.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:17 smithi089 bash[21793]: cluster 2024-07-15T06:55:16.521574+0000 mgr.x (mgr.24544) 17 : cluster [DBG] pgmap v11: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-07-15T06:55:18.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:17 smithi089 bash[21793]: audit 2024-07-15T06:55:17.416423+0000 mon.a (mon.0) 1013 : audit [DBG] from='client.? 172.21.15.89:0/2236019436' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:55:18.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:17 smithi089 bash[31372]: cluster 2024-07-15T06:55:16.521574+0000 mgr.x (mgr.24544) 17 : cluster [DBG] pgmap v11: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-07-15T06:55:18.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:17 smithi089 bash[31372]: audit 2024-07-15T06:55:17.416423+0000 mon.a (mon.0) 1013 : audit [DBG] from='client.? 172.21.15.89:0/2236019436' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:55:20.005 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:55:19 smithi089 systemd[1]: Stopping Ceph node-exporter.a for 75ba02bc-4274-11ef-bcad-c7b262605968... 2024-07-15T06:55:20.005 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:19 smithi089 bash[31372]: cluster 2024-07-15T06:55:18.522696+0000 mgr.x (mgr.24544) 18 : cluster [DBG] pgmap v12: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-07-15T06:55:20.006 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:19 smithi089 bash[31372]: audit 2024-07-15T06:55:18.632984+0000 mon.a (mon.0) 1014 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:20.006 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:19 smithi089 bash[31372]: audit 2024-07-15T06:55:18.647046+0000 mon.a (mon.0) 1015 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:20.006 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:19 smithi089 bash[31372]: audit 2024-07-15T06:55:18.650985+0000 mon.b (mon.1) 61 : audit [INF] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:55:20.006 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:19 smithi089 bash[31372]: audit 2024-07-15T06:55:18.651418+0000 mon.a (mon.0) 1016 : audit [INF] from='mgr.24544 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:55:20.006 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:19 smithi089 bash[31372]: audit 2024-07-15T06:55:18.653759+0000 mon.b (mon.1) 62 : audit [INF] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:55:20.006 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:19 smithi089 bash[31372]: audit 2024-07-15T06:55:18.654123+0000 mon.a (mon.0) 1017 : audit [INF] from='mgr.24544 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:55:20.006 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:19 smithi089 bash[31372]: audit 2024-07-15T06:55:18.656534+0000 mon.b (mon.1) 63 : audit [INF] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:55:20.006 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:19 smithi089 bash[31372]: audit 2024-07-15T06:55:18.656887+0000 mon.a (mon.0) 1018 : audit [INF] from='mgr.24544 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:55:20.006 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:19 smithi089 bash[31372]: audit 2024-07-15T06:55:18.659455+0000 mon.b (mon.1) 64 : audit [INF] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:55:20.006 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:19 smithi089 bash[31372]: audit 2024-07-15T06:55:18.659780+0000 mon.a (mon.0) 1019 : audit [INF] from='mgr.24544 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:55:20.006 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:19 smithi089 bash[31372]: cephadm 2024-07-15T06:55:18.661884+0000 mgr.x (mgr.24544) 19 : cephadm [INF] Adjusting osd_memory_target on smithi134 to 2524M 2024-07-15T06:55:20.006 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:19 smithi089 bash[31372]: audit 2024-07-15T06:55:18.670433+0000 mon.a (mon.0) 1020 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:20.007 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:19 smithi089 bash[31372]: audit 2024-07-15T06:55:18.674537+0000 mon.b (mon.1) 65 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:55:20.007 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:19 smithi089 bash[31372]: audit 2024-07-15T06:55:18.676253+0000 mon.b (mon.1) 66 : audit [INF] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:55:20.007 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:19 smithi089 bash[31372]: cephadm 2024-07-15T06:55:18.678684+0000 mgr.x (mgr.24544) 20 : cephadm [INF] Updating smithi089:/etc/ceph/ceph.conf 2024-07-15T06:55:20.007 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:19 smithi089 bash[31372]: cephadm 2024-07-15T06:55:18.679071+0000 mgr.x (mgr.24544) 21 : cephadm [INF] Updating smithi134:/etc/ceph/ceph.conf 2024-07-15T06:55:20.007 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:19 smithi089 bash[31372]: audit 2024-07-15T06:55:19.282427+0000 mon.a (mon.0) 1021 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:20.007 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:19 smithi089 bash[31372]: audit 2024-07-15T06:55:19.295235+0000 mon.a (mon.0) 1022 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:20.007 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:19 smithi089 bash[31372]: audit 2024-07-15T06:55:19.301980+0000 mon.a (mon.0) 1023 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:20.007 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:19 smithi089 bash[31372]: audit 2024-07-15T06:55:19.311631+0000 mon.a (mon.0) 1024 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:20.007 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:19 smithi089 bash[21793]: cluster 2024-07-15T06:55:18.522696+0000 mgr.x (mgr.24544) 18 : cluster [DBG] pgmap v12: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-07-15T06:55:20.007 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:19 smithi089 bash[21793]: audit 2024-07-15T06:55:18.632984+0000 mon.a (mon.0) 1014 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:20.008 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:19 smithi089 bash[21793]: audit 2024-07-15T06:55:18.647046+0000 mon.a (mon.0) 1015 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:20.008 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:19 smithi089 bash[21793]: audit 2024-07-15T06:55:18.650985+0000 mon.b (mon.1) 61 : audit [INF] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:55:20.008 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:19 smithi089 bash[21793]: audit 2024-07-15T06:55:18.651418+0000 mon.a (mon.0) 1016 : audit [INF] from='mgr.24544 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:55:20.008 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:19 smithi089 bash[21793]: audit 2024-07-15T06:55:18.653759+0000 mon.b (mon.1) 62 : audit [INF] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:55:20.008 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:19 smithi089 bash[21793]: audit 2024-07-15T06:55:18.654123+0000 mon.a (mon.0) 1017 : audit [INF] from='mgr.24544 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:55:20.008 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:19 smithi089 bash[21793]: audit 2024-07-15T06:55:18.656534+0000 mon.b (mon.1) 63 : audit [INF] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:55:20.008 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:19 smithi089 bash[21793]: audit 2024-07-15T06:55:18.656887+0000 mon.a (mon.0) 1018 : audit [INF] from='mgr.24544 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:55:20.008 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:19 smithi089 bash[21793]: audit 2024-07-15T06:55:18.659455+0000 mon.b (mon.1) 64 : audit [INF] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:55:20.008 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:19 smithi089 bash[21793]: audit 2024-07-15T06:55:18.659780+0000 mon.a (mon.0) 1019 : audit [INF] from='mgr.24544 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:55:20.008 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:19 smithi089 bash[21793]: cephadm 2024-07-15T06:55:18.661884+0000 mgr.x (mgr.24544) 19 : cephadm [INF] Adjusting osd_memory_target on smithi134 to 2524M 2024-07-15T06:55:20.008 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:19 smithi089 bash[21793]: audit 2024-07-15T06:55:18.670433+0000 mon.a (mon.0) 1020 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:20.008 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:19 smithi089 bash[21793]: audit 2024-07-15T06:55:18.674537+0000 mon.b (mon.1) 65 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:55:20.008 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:19 smithi089 bash[21793]: audit 2024-07-15T06:55:18.676253+0000 mon.b (mon.1) 66 : audit [INF] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:55:20.009 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:19 smithi089 bash[21793]: cephadm 2024-07-15T06:55:18.678684+0000 mgr.x (mgr.24544) 20 : cephadm [INF] Updating smithi089:/etc/ceph/ceph.conf 2024-07-15T06:55:20.009 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:19 smithi089 bash[21793]: cephadm 2024-07-15T06:55:18.679071+0000 mgr.x (mgr.24544) 21 : cephadm [INF] Updating smithi134:/etc/ceph/ceph.conf 2024-07-15T06:55:20.009 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:19 smithi089 bash[21793]: audit 2024-07-15T06:55:19.282427+0000 mon.a (mon.0) 1021 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:20.009 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:19 smithi089 bash[21793]: audit 2024-07-15T06:55:19.295235+0000 mon.a (mon.0) 1022 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:20.009 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:19 smithi089 bash[21793]: audit 2024-07-15T06:55:19.301980+0000 mon.a (mon.0) 1023 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:20.009 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:19 smithi089 bash[21793]: audit 2024-07-15T06:55:19.311631+0000 mon.a (mon.0) 1024 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:20.009 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:19 smithi089 bash[31372]: audit 2024-07-15T06:55:19.320795+0000 mon.a (mon.0) 1025 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:20.009 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:19 smithi089 bash[31372]: audit 2024-07-15T06:55:19.363488+0000 mon.a (mon.0) 1026 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:20.009 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:19 smithi089 bash[31372]: audit 2024-07-15T06:55:19.370627+0000 mon.a (mon.0) 1027 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:20.009 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:19 smithi089 bash[31372]: audit 2024-07-15T06:55:19.377151+0000 mon.a (mon.0) 1028 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:20.010 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:19 smithi089 bash[31372]: audit 2024-07-15T06:55:19.384064+0000 mon.a (mon.0) 1029 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:20.010 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:19 smithi089 bash[21793]: audit 2024-07-15T06:55:19.320795+0000 mon.a (mon.0) 1025 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:20.010 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:19 smithi089 bash[21793]: audit 2024-07-15T06:55:19.363488+0000 mon.a (mon.0) 1026 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:20.010 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:19 smithi089 bash[21793]: audit 2024-07-15T06:55:19.370627+0000 mon.a (mon.0) 1027 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:20.010 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:19 smithi089 bash[21793]: audit 2024-07-15T06:55:19.377151+0000 mon.a (mon.0) 1028 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:20.010 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:19 smithi089 bash[21793]: audit 2024-07-15T06:55:19.384064+0000 mon.a (mon.0) 1029 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:20.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:19 smithi134 bash[24068]: cluster 2024-07-15T06:55:18.522696+0000 mgr.x (mgr.24544) 18 : cluster [DBG] pgmap v12: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-07-15T06:55:20.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:19 smithi134 bash[24068]: audit 2024-07-15T06:55:18.632984+0000 mon.a (mon.0) 1014 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:20.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:19 smithi134 bash[24068]: audit 2024-07-15T06:55:18.647046+0000 mon.a (mon.0) 1015 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:20.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:19 smithi134 bash[24068]: audit 2024-07-15T06:55:18.650985+0000 mon.b (mon.1) 61 : audit [INF] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:55:20.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:19 smithi134 bash[24068]: audit 2024-07-15T06:55:18.651418+0000 mon.a (mon.0) 1016 : audit [INF] from='mgr.24544 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:55:20.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:19 smithi134 bash[24068]: audit 2024-07-15T06:55:18.653759+0000 mon.b (mon.1) 62 : audit [INF] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:55:20.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:19 smithi134 bash[24068]: audit 2024-07-15T06:55:18.654123+0000 mon.a (mon.0) 1017 : audit [INF] from='mgr.24544 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:55:20.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:19 smithi134 bash[24068]: audit 2024-07-15T06:55:18.656534+0000 mon.b (mon.1) 63 : audit [INF] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:55:20.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:19 smithi134 bash[24068]: audit 2024-07-15T06:55:18.656887+0000 mon.a (mon.0) 1018 : audit [INF] from='mgr.24544 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:55:20.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:19 smithi134 bash[24068]: audit 2024-07-15T06:55:18.659455+0000 mon.b (mon.1) 64 : audit [INF] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:55:20.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:19 smithi134 bash[24068]: audit 2024-07-15T06:55:18.659780+0000 mon.a (mon.0) 1019 : audit [INF] from='mgr.24544 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-07-15T06:55:20.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:19 smithi134 bash[24068]: cephadm 2024-07-15T06:55:18.661884+0000 mgr.x (mgr.24544) 19 : cephadm [INF] Adjusting osd_memory_target on smithi134 to 2524M 2024-07-15T06:55:20.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:19 smithi134 bash[24068]: audit 2024-07-15T06:55:18.670433+0000 mon.a (mon.0) 1020 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:20.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:19 smithi134 bash[24068]: audit 2024-07-15T06:55:18.674537+0000 mon.b (mon.1) 65 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:55:20.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:19 smithi134 bash[24068]: audit 2024-07-15T06:55:18.676253+0000 mon.b (mon.1) 66 : audit [INF] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:55:20.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:19 smithi134 bash[24068]: cephadm 2024-07-15T06:55:18.678684+0000 mgr.x (mgr.24544) 20 : cephadm [INF] Updating smithi089:/etc/ceph/ceph.conf 2024-07-15T06:55:20.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:19 smithi134 bash[24068]: cephadm 2024-07-15T06:55:18.679071+0000 mgr.x (mgr.24544) 21 : cephadm [INF] Updating smithi134:/etc/ceph/ceph.conf 2024-07-15T06:55:20.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:19 smithi134 bash[24068]: audit 2024-07-15T06:55:19.282427+0000 mon.a (mon.0) 1021 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:20.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:19 smithi134 bash[24068]: audit 2024-07-15T06:55:19.295235+0000 mon.a (mon.0) 1022 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:20.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:19 smithi134 bash[24068]: audit 2024-07-15T06:55:19.301980+0000 mon.a (mon.0) 1023 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:20.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:19 smithi134 bash[24068]: audit 2024-07-15T06:55:19.311631+0000 mon.a (mon.0) 1024 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:20.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:19 smithi134 bash[24068]: audit 2024-07-15T06:55:19.320795+0000 mon.a (mon.0) 1025 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:20.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:19 smithi134 bash[24068]: audit 2024-07-15T06:55:19.363488+0000 mon.a (mon.0) 1026 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:20.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:19 smithi134 bash[24068]: audit 2024-07-15T06:55:19.370627+0000 mon.a (mon.0) 1027 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:20.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:19 smithi134 bash[24068]: audit 2024-07-15T06:55:19.377151+0000 mon.a (mon.0) 1028 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:20.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:19 smithi134 bash[24068]: audit 2024-07-15T06:55:19.384064+0000 mon.a (mon.0) 1029 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:20.675 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:55:20 smithi089 bash[71845]: ceph-75ba02bc-4274-11ef-bcad-c7b262605968-node-exporter-a 2024-07-15T06:55:20.675 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:55:20 smithi089 systemd[1]: ceph-75ba02bc-4274-11ef-bcad-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-07-15T06:55:20.675 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:55:20 smithi089 bash[71909]: Error response from daemon: No such container: ceph-75ba02bc-4274-11ef-bcad-c7b262605968-node-exporter-a 2024-07-15T06:55:20.675 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:55:20 smithi089 systemd[1]: ceph-75ba02bc-4274-11ef-bcad-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-07-15T06:55:20.675 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:55:20 smithi089 systemd[1]: Stopped Ceph node-exporter.a for 75ba02bc-4274-11ef-bcad-c7b262605968. 2024-07-15T06:55:20.675 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:55:20 smithi089 systemd[1]: Started Ceph node-exporter.a for 75ba02bc-4274-11ef-bcad-c7b262605968. 2024-07-15T06:55:20.676 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:20 smithi089 bash[21793]: cephadm 2024-07-15T06:55:18.869875+0000 mgr.x (mgr.24544) 22 : cephadm [INF] Updating smithi089:/var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/config/ceph.conf 2024-07-15T06:55:20.676 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:20 smithi089 bash[21793]: cephadm 2024-07-15T06:55:18.870066+0000 mgr.x (mgr.24544) 23 : cephadm [INF] Updating smithi134:/var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/config/ceph.conf 2024-07-15T06:55:20.676 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:20 smithi089 bash[21793]: cephadm 2024-07-15T06:55:19.010646+0000 mgr.x (mgr.24544) 24 : cephadm [INF] Updating smithi089:/etc/ceph/ceph.client.admin.keyring 2024-07-15T06:55:20.676 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:20 smithi089 bash[21793]: cephadm 2024-07-15T06:55:19.010863+0000 mgr.x (mgr.24544) 25 : cephadm [INF] Updating smithi134:/etc/ceph/ceph.client.admin.keyring 2024-07-15T06:55:20.676 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:20 smithi089 bash[21793]: cephadm 2024-07-15T06:55:19.149219+0000 mgr.x (mgr.24544) 26 : cephadm [INF] Updating smithi134:/var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/config/ceph.client.admin.keyring 2024-07-15T06:55:20.676 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:20 smithi089 bash[21793]: cephadm 2024-07-15T06:55:19.152025+0000 mgr.x (mgr.24544) 27 : cephadm [INF] Updating smithi089:/var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/config/ceph.client.admin.keyring 2024-07-15T06:55:20.676 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:20 smithi089 bash[21793]: cephadm 2024-07-15T06:55:19.386221+0000 mgr.x (mgr.24544) 28 : cephadm [INF] Reconfiguring node-exporter.a (dependencies changed)... 2024-07-15T06:55:20.676 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:20 smithi089 bash[21793]: cephadm 2024-07-15T06:55:19.386806+0000 mgr.x (mgr.24544) 29 : cephadm [INF] Reconfiguring daemon node-exporter.a on smithi089 2024-07-15T06:55:20.676 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:20 smithi089 bash[21793]: audit 2024-07-15T06:55:19.804531+0000 mon.c (mon.2) 45 : audit [DBG] from='client.? 172.21.15.89:0/3883873804' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:55:20.676 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:20 smithi089 bash[31372]: cephadm 2024-07-15T06:55:18.869875+0000 mgr.x (mgr.24544) 22 : cephadm [INF] Updating smithi089:/var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/config/ceph.conf 2024-07-15T06:55:20.677 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:20 smithi089 bash[31372]: cephadm 2024-07-15T06:55:18.870066+0000 mgr.x (mgr.24544) 23 : cephadm [INF] Updating smithi134:/var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/config/ceph.conf 2024-07-15T06:55:20.677 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:20 smithi089 bash[31372]: cephadm 2024-07-15T06:55:19.010646+0000 mgr.x (mgr.24544) 24 : cephadm [INF] Updating smithi089:/etc/ceph/ceph.client.admin.keyring 2024-07-15T06:55:20.677 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:20 smithi089 bash[31372]: cephadm 2024-07-15T06:55:19.010863+0000 mgr.x (mgr.24544) 25 : cephadm [INF] Updating smithi134:/etc/ceph/ceph.client.admin.keyring 2024-07-15T06:55:20.677 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:20 smithi089 bash[31372]: cephadm 2024-07-15T06:55:19.149219+0000 mgr.x (mgr.24544) 26 : cephadm [INF] Updating smithi134:/var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/config/ceph.client.admin.keyring 2024-07-15T06:55:20.677 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:20 smithi089 bash[31372]: cephadm 2024-07-15T06:55:19.152025+0000 mgr.x (mgr.24544) 27 : cephadm [INF] Updating smithi089:/var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/config/ceph.client.admin.keyring 2024-07-15T06:55:20.677 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:20 smithi089 bash[31372]: cephadm 2024-07-15T06:55:19.386221+0000 mgr.x (mgr.24544) 28 : cephadm [INF] Reconfiguring node-exporter.a (dependencies changed)... 2024-07-15T06:55:20.677 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:20 smithi089 bash[31372]: cephadm 2024-07-15T06:55:19.386806+0000 mgr.x (mgr.24544) 29 : cephadm [INF] Reconfiguring daemon node-exporter.a on smithi089 2024-07-15T06:55:20.677 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:20 smithi089 bash[31372]: audit 2024-07-15T06:55:19.804531+0000 mon.c (mon.2) 45 : audit [DBG] from='client.? 172.21.15.89:0/3883873804' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:55:21.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:20 smithi134 bash[24068]: cephadm 2024-07-15T06:55:18.869875+0000 mgr.x (mgr.24544) 22 : cephadm [INF] Updating smithi089:/var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/config/ceph.conf 2024-07-15T06:55:21.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:20 smithi134 bash[24068]: cephadm 2024-07-15T06:55:18.870066+0000 mgr.x (mgr.24544) 23 : cephadm [INF] Updating smithi134:/var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/config/ceph.conf 2024-07-15T06:55:21.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:20 smithi134 bash[24068]: cephadm 2024-07-15T06:55:19.010646+0000 mgr.x (mgr.24544) 24 : cephadm [INF] Updating smithi089:/etc/ceph/ceph.client.admin.keyring 2024-07-15T06:55:21.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:20 smithi134 bash[24068]: cephadm 2024-07-15T06:55:19.010863+0000 mgr.x (mgr.24544) 25 : cephadm [INF] Updating smithi134:/etc/ceph/ceph.client.admin.keyring 2024-07-15T06:55:21.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:20 smithi134 bash[24068]: cephadm 2024-07-15T06:55:19.149219+0000 mgr.x (mgr.24544) 26 : cephadm [INF] Updating smithi134:/var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/config/ceph.client.admin.keyring 2024-07-15T06:55:21.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:20 smithi134 bash[24068]: cephadm 2024-07-15T06:55:19.152025+0000 mgr.x (mgr.24544) 27 : cephadm [INF] Updating smithi089:/var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/config/ceph.client.admin.keyring 2024-07-15T06:55:21.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:20 smithi134 bash[24068]: cephadm 2024-07-15T06:55:19.386221+0000 mgr.x (mgr.24544) 28 : cephadm [INF] Reconfiguring node-exporter.a (dependencies changed)... 2024-07-15T06:55:21.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:20 smithi134 bash[24068]: cephadm 2024-07-15T06:55:19.386806+0000 mgr.x (mgr.24544) 29 : cephadm [INF] Reconfiguring daemon node-exporter.a on smithi089 2024-07-15T06:55:21.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:20 smithi134 bash[24068]: audit 2024-07-15T06:55:19.804531+0000 mon.c (mon.2) 45 : audit [DBG] from='client.? 172.21.15.89:0/3883873804' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:55:22.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:21 smithi134 bash[24068]: cluster 2024-07-15T06:55:20.523379+0000 mgr.x (mgr.24544) 30 : cluster [DBG] pgmap v13: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:55:22.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:21 smithi134 bash[24068]: audit 2024-07-15T06:55:20.720193+0000 mon.a (mon.0) 1030 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:22.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:21 smithi134 bash[24068]: audit 2024-07-15T06:55:20.732695+0000 mon.a (mon.0) 1031 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:22.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:21 smithi134 bash[24068]: cephadm 2024-07-15T06:55:20.739189+0000 mgr.x (mgr.24544) 31 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2024-07-15T06:55:22.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:21 smithi134 bash[24068]: cephadm 2024-07-15T06:55:20.766807+0000 mgr.x (mgr.24544) 32 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi089 2024-07-15T06:55:22.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:21 smithi089 bash[21793]: cluster 2024-07-15T06:55:20.523379+0000 mgr.x (mgr.24544) 30 : cluster [DBG] pgmap v13: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:55:22.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:21 smithi089 bash[21793]: audit 2024-07-15T06:55:20.720193+0000 mon.a (mon.0) 1030 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:22.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:21 smithi089 bash[21793]: audit 2024-07-15T06:55:20.732695+0000 mon.a (mon.0) 1031 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:22.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:21 smithi089 bash[21793]: cephadm 2024-07-15T06:55:20.739189+0000 mgr.x (mgr.24544) 31 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2024-07-15T06:55:22.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:21 smithi089 bash[21793]: cephadm 2024-07-15T06:55:20.766807+0000 mgr.x (mgr.24544) 32 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi089 2024-07-15T06:55:22.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:21 smithi089 bash[31372]: cluster 2024-07-15T06:55:20.523379+0000 mgr.x (mgr.24544) 30 : cluster [DBG] pgmap v13: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:55:22.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:21 smithi089 bash[31372]: audit 2024-07-15T06:55:20.720193+0000 mon.a (mon.0) 1030 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:22.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:21 smithi089 bash[31372]: audit 2024-07-15T06:55:20.732695+0000 mon.a (mon.0) 1031 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:22.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:21 smithi089 bash[31372]: cephadm 2024-07-15T06:55:20.739189+0000 mgr.x (mgr.24544) 31 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2024-07-15T06:55:22.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:21 smithi089 bash[31372]: cephadm 2024-07-15T06:55:20.766807+0000 mgr.x (mgr.24544) 32 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi089 2024-07-15T06:55:22.898 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:22 smithi089 bash[21793]: audit 2024-07-15T06:55:22.177262+0000 mon.c (mon.2) 46 : audit [DBG] from='client.? 172.21.15.89:0/1745731672' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:55:22.899 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:22 smithi089 bash[31372]: audit 2024-07-15T06:55:22.177262+0000 mon.c (mon.2) 46 : audit [DBG] from='client.? 172.21.15.89:0/1745731672' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:55:22.899 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:55:22 smithi089 bash[22045]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:55:22] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-07-15T06:55:22.900 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:55:22 smithi089 bash[71937]: ts=2024-07-15T06:55:22.797Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-07-15T06:55:22.900 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:55:22 smithi089 bash[71937]: ts=2024-07-15T06:55:22.797Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-07-15T06:55:22.900 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:55:22 smithi089 bash[71937]: ts=2024-07-15T06:55:22.797Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-07-15T06:55:22.900 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:55:22 smithi089 bash[71937]: ts=2024-07-15T06:55:22.797Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-07-15T06:55:22.900 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:55:22 smithi089 bash[71937]: ts=2024-07-15T06:55:22.798Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-07-15T06:55:22.900 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:55:22 smithi089 bash[71937]: ts=2024-07-15T06:55:22.798Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-07-15T06:55:22.900 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:55:22 smithi089 bash[71937]: ts=2024-07-15T06:55:22.798Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-07-15T06:55:22.900 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:55:22 smithi089 bash[71937]: ts=2024-07-15T06:55:22.798Z caller=node_exporter.go:117 level=info collector=arp 2024-07-15T06:55:22.900 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:55:22 smithi089 bash[71937]: ts=2024-07-15T06:55:22.798Z caller=node_exporter.go:117 level=info collector=bcache 2024-07-15T06:55:22.901 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:55:22 smithi089 bash[71937]: ts=2024-07-15T06:55:22.798Z caller=node_exporter.go:117 level=info collector=bonding 2024-07-15T06:55:22.901 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:55:22 smithi089 bash[71937]: ts=2024-07-15T06:55:22.798Z caller=node_exporter.go:117 level=info collector=btrfs 2024-07-15T06:55:22.901 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:55:22 smithi089 bash[71937]: ts=2024-07-15T06:55:22.798Z caller=node_exporter.go:117 level=info collector=conntrack 2024-07-15T06:55:22.901 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:55:22 smithi089 bash[71937]: ts=2024-07-15T06:55:22.798Z caller=node_exporter.go:117 level=info collector=cpu 2024-07-15T06:55:22.901 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:55:22 smithi089 bash[71937]: ts=2024-07-15T06:55:22.798Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-07-15T06:55:22.901 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:55:22 smithi089 bash[71937]: ts=2024-07-15T06:55:22.798Z caller=node_exporter.go:117 level=info collector=diskstats 2024-07-15T06:55:22.901 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:55:22 smithi089 bash[71937]: ts=2024-07-15T06:55:22.798Z caller=node_exporter.go:117 level=info collector=dmi 2024-07-15T06:55:22.901 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:55:22 smithi089 bash[71937]: ts=2024-07-15T06:55:22.798Z caller=node_exporter.go:117 level=info collector=edac 2024-07-15T06:55:22.901 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:55:22 smithi089 bash[71937]: ts=2024-07-15T06:55:22.798Z caller=node_exporter.go:117 level=info collector=entropy 2024-07-15T06:55:22.901 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:55:22 smithi089 bash[71937]: ts=2024-07-15T06:55:22.798Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-07-15T06:55:22.901 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:55:22 smithi089 bash[71937]: ts=2024-07-15T06:55:22.798Z caller=node_exporter.go:117 level=info collector=filefd 2024-07-15T06:55:22.901 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:55:22 smithi089 bash[71937]: ts=2024-07-15T06:55:22.798Z caller=node_exporter.go:117 level=info collector=filesystem 2024-07-15T06:55:22.901 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:55:22 smithi089 bash[71937]: ts=2024-07-15T06:55:22.798Z caller=node_exporter.go:117 level=info collector=hwmon 2024-07-15T06:55:22.902 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:55:22 smithi089 bash[71937]: ts=2024-07-15T06:55:22.798Z caller=node_exporter.go:117 level=info collector=infiniband 2024-07-15T06:55:22.902 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:55:22 smithi089 bash[71937]: ts=2024-07-15T06:55:22.798Z caller=node_exporter.go:117 level=info collector=ipvs 2024-07-15T06:55:22.902 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:55:22 smithi089 bash[71937]: ts=2024-07-15T06:55:22.798Z caller=node_exporter.go:117 level=info collector=loadavg 2024-07-15T06:55:22.902 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:55:22 smithi089 bash[71937]: ts=2024-07-15T06:55:22.798Z caller=node_exporter.go:117 level=info collector=mdadm 2024-07-15T06:55:22.902 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:55:22 smithi089 bash[71937]: ts=2024-07-15T06:55:22.798Z caller=node_exporter.go:117 level=info collector=meminfo 2024-07-15T06:55:22.902 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:55:22 smithi089 bash[71937]: ts=2024-07-15T06:55:22.798Z caller=node_exporter.go:117 level=info collector=netclass 2024-07-15T06:55:22.902 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:55:22 smithi089 bash[71937]: ts=2024-07-15T06:55:22.798Z caller=node_exporter.go:117 level=info collector=netdev 2024-07-15T06:55:22.902 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:55:22 smithi089 bash[71937]: ts=2024-07-15T06:55:22.798Z caller=node_exporter.go:117 level=info collector=netstat 2024-07-15T06:55:22.902 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:55:22 smithi089 bash[71937]: ts=2024-07-15T06:55:22.798Z caller=node_exporter.go:117 level=info collector=nfs 2024-07-15T06:55:22.902 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:55:22 smithi089 bash[71937]: ts=2024-07-15T06:55:22.798Z caller=node_exporter.go:117 level=info collector=nfsd 2024-07-15T06:55:22.902 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:55:22 smithi089 bash[71937]: ts=2024-07-15T06:55:22.798Z caller=node_exporter.go:117 level=info collector=nvme 2024-07-15T06:55:22.902 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:55:22 smithi089 bash[71937]: ts=2024-07-15T06:55:22.798Z caller=node_exporter.go:117 level=info collector=os 2024-07-15T06:55:22.903 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:55:22 smithi089 bash[71937]: ts=2024-07-15T06:55:22.798Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-07-15T06:55:22.903 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:55:22 smithi089 bash[71937]: ts=2024-07-15T06:55:22.798Z caller=node_exporter.go:117 level=info collector=pressure 2024-07-15T06:55:22.903 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:55:22 smithi089 bash[71937]: ts=2024-07-15T06:55:22.798Z caller=node_exporter.go:117 level=info collector=rapl 2024-07-15T06:55:22.903 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:55:22 smithi089 bash[71937]: ts=2024-07-15T06:55:22.798Z caller=node_exporter.go:117 level=info collector=schedstat 2024-07-15T06:55:22.903 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:55:22 smithi089 bash[71937]: ts=2024-07-15T06:55:22.798Z caller=node_exporter.go:117 level=info collector=selinux 2024-07-15T06:55:22.903 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:55:22 smithi089 bash[71937]: ts=2024-07-15T06:55:22.798Z caller=node_exporter.go:117 level=info collector=sockstat 2024-07-15T06:55:22.903 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:55:22 smithi089 bash[71937]: ts=2024-07-15T06:55:22.798Z caller=node_exporter.go:117 level=info collector=softnet 2024-07-15T06:55:22.903 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:55:22 smithi089 bash[71937]: ts=2024-07-15T06:55:22.798Z caller=node_exporter.go:117 level=info collector=stat 2024-07-15T06:55:22.903 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:55:22 smithi089 bash[71937]: ts=2024-07-15T06:55:22.798Z caller=node_exporter.go:117 level=info collector=tapestats 2024-07-15T06:55:22.903 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:55:22 smithi089 bash[71937]: ts=2024-07-15T06:55:22.798Z caller=node_exporter.go:117 level=info collector=textfile 2024-07-15T06:55:22.903 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:55:22 smithi089 bash[71937]: ts=2024-07-15T06:55:22.798Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-07-15T06:55:22.903 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:55:22 smithi089 bash[71937]: ts=2024-07-15T06:55:22.798Z caller=node_exporter.go:117 level=info collector=time 2024-07-15T06:55:22.903 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:55:22 smithi089 bash[71937]: ts=2024-07-15T06:55:22.798Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-07-15T06:55:22.903 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:55:22 smithi089 bash[71937]: ts=2024-07-15T06:55:22.798Z caller=node_exporter.go:117 level=info collector=uname 2024-07-15T06:55:22.903 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:55:22 smithi089 bash[71937]: ts=2024-07-15T06:55:22.798Z caller=node_exporter.go:117 level=info collector=vmstat 2024-07-15T06:55:22.904 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:55:22 smithi089 bash[71937]: ts=2024-07-15T06:55:22.798Z caller=node_exporter.go:117 level=info collector=xfs 2024-07-15T06:55:22.904 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:55:22 smithi089 bash[71937]: ts=2024-07-15T06:55:22.798Z caller=node_exporter.go:117 level=info collector=zfs 2024-07-15T06:55:22.904 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:55:22 smithi089 bash[71937]: ts=2024-07-15T06:55:22.799Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-07-15T06:55:22.904 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:55:22 smithi089 bash[71937]: ts=2024-07-15T06:55:22.799Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-07-15T06:55:23.053 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:22 smithi134 bash[24068]: audit 2024-07-15T06:55:22.177262+0000 mon.c (mon.2) 46 : audit [DBG] from='client.? 172.21.15.89:0/1745731672' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:55:24.053 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:23 smithi134 bash[24068]: cluster 2024-07-15T06:55:22.524467+0000 mgr.x (mgr.24544) 33 : cluster [DBG] pgmap v14: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:55:24.105 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:23 smithi089 bash[21793]: cluster 2024-07-15T06:55:22.524467+0000 mgr.x (mgr.24544) 33 : cluster [DBG] pgmap v14: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:55:24.105 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:23 smithi089 bash[31372]: cluster 2024-07-15T06:55:22.524467+0000 mgr.x (mgr.24544) 33 : cluster [DBG] pgmap v14: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:55:25.012 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:24 smithi089 bash[21793]: audit 2024-07-15T06:55:24.538475+0000 mon.a (mon.0) 1032 : audit [DBG] from='client.? 172.21.15.89:0/78395595' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:55:25.012 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:24 smithi089 bash[31372]: audit 2024-07-15T06:55:24.538475+0000 mon.a (mon.0) 1032 : audit [DBG] from='client.? 172.21.15.89:0/78395595' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:55:25.053 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:24 smithi134 bash[24068]: audit 2024-07-15T06:55:24.538475+0000 mon.a (mon.0) 1032 : audit [DBG] from='client.? 172.21.15.89:0/78395595' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:55:25.846 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:25 smithi089 bash[21793]: cluster 2024-07-15T06:55:24.525162+0000 mgr.x (mgr.24544) 34 : cluster [DBG] pgmap v15: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:55:25.847 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:25 smithi089 bash[31372]: cluster 2024-07-15T06:55:24.525162+0000 mgr.x (mgr.24544) 34 : cluster [DBG] pgmap v15: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:55:25.847 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 06:55:25 smithi089 systemd[1]: Stopping Ceph alertmanager.a for 75ba02bc-4274-11ef-bcad-c7b262605968... 2024-07-15T06:55:25.847 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 06:55:25 smithi089 bash[58097]: ts=2024-07-15T06:55:25.669Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-07-15T06:55:26.053 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:25 smithi134 bash[24068]: cluster 2024-07-15T06:55:24.525162+0000 mgr.x (mgr.24544) 34 : cluster [DBG] pgmap v15: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:55:26.560 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 06:55:26 smithi089 bash[72219]: ceph-75ba02bc-4274-11ef-bcad-c7b262605968-alertmanager-a 2024-07-15T06:55:26.561 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 06:55:26 smithi089 bash[72280]: Error response from daemon: No such container: ceph-75ba02bc-4274-11ef-bcad-c7b262605968-alertmanager-a 2024-07-15T06:55:26.561 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 06:55:26 smithi089 systemd[1]: ceph-75ba02bc-4274-11ef-bcad-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-07-15T06:55:26.561 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 06:55:26 smithi089 systemd[1]: Stopped Ceph alertmanager.a for 75ba02bc-4274-11ef-bcad-c7b262605968. 2024-07-15T06:55:26.561 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 06:55:26 smithi089 systemd[1]: Started Ceph alertmanager.a for 75ba02bc-4274-11ef-bcad-c7b262605968. 2024-07-15T06:55:27.303 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:55:26 smithi134 bash[52600]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:55:26] "GET /metrics HTTP/1.1" 200 33807 "" "Prometheus/2.43.0" 2024-07-15T06:55:27.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:27 smithi134 bash[24068]: audit 2024-07-15T06:55:26.465059+0000 mon.a (mon.0) 1033 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:27.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:27 smithi134 bash[24068]: audit 2024-07-15T06:55:26.477077+0000 mon.a (mon.0) 1034 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:27.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:27 smithi134 bash[24068]: cephadm 2024-07-15T06:55:26.483854+0000 mgr.x (mgr.24544) 35 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2024-07-15T06:55:27.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:27 smithi134 bash[24068]: cluster 2024-07-15T06:55:26.525793+0000 mgr.x (mgr.24544) 36 : cluster [DBG] pgmap v16: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:55:27.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:27 smithi134 bash[24068]: cephadm 2024-07-15T06:55:26.729817+0000 mgr.x (mgr.24544) 37 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi134 2024-07-15T06:55:27.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:27 smithi134 bash[24068]: audit 2024-07-15T06:55:26.896105+0000 mon.c (mon.2) 47 : audit [DBG] from='client.? 172.21.15.89:0/2265601922' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:55:27.869 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:27 smithi089 bash[21793]: audit 2024-07-15T06:55:26.465059+0000 mon.a (mon.0) 1033 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:27.869 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:27 smithi089 bash[21793]: audit 2024-07-15T06:55:26.477077+0000 mon.a (mon.0) 1034 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:27.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:27 smithi089 bash[21793]: cephadm 2024-07-15T06:55:26.483854+0000 mgr.x (mgr.24544) 35 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2024-07-15T06:55:27.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:27 smithi089 bash[21793]: cluster 2024-07-15T06:55:26.525793+0000 mgr.x (mgr.24544) 36 : cluster [DBG] pgmap v16: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:55:27.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:27 smithi089 bash[21793]: cephadm 2024-07-15T06:55:26.729817+0000 mgr.x (mgr.24544) 37 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi134 2024-07-15T06:55:27.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:27 smithi089 bash[21793]: audit 2024-07-15T06:55:26.896105+0000 mon.c (mon.2) 47 : audit [DBG] from='client.? 172.21.15.89:0/2265601922' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:55:27.870 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:27 smithi089 bash[31372]: audit 2024-07-15T06:55:26.465059+0000 mon.a (mon.0) 1033 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:27.870 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:27 smithi089 bash[31372]: audit 2024-07-15T06:55:26.477077+0000 mon.a (mon.0) 1034 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:27.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:27 smithi089 bash[31372]: cephadm 2024-07-15T06:55:26.483854+0000 mgr.x (mgr.24544) 35 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2024-07-15T06:55:27.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:27 smithi089 bash[31372]: cluster 2024-07-15T06:55:26.525793+0000 mgr.x (mgr.24544) 36 : cluster [DBG] pgmap v16: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:55:27.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:27 smithi089 bash[31372]: cephadm 2024-07-15T06:55:26.729817+0000 mgr.x (mgr.24544) 37 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi134 2024-07-15T06:55:27.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:27 smithi089 bash[31372]: audit 2024-07-15T06:55:26.896105+0000 mon.c (mon.2) 47 : audit [DBG] from='client.? 172.21.15.89:0/2265601922' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:55:27.871 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 06:55:27 smithi089 bash[72306]: ts=2024-07-15T06:55:27.626Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-07-15T06:55:27.871 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 06:55:27 smithi089 bash[72306]: ts=2024-07-15T06:55:27.626Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-07-15T06:55:27.871 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 06:55:27 smithi089 bash[72306]: ts=2024-07-15T06:55:27.628Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.89 port=9094 2024-07-15T06:55:27.872 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 06:55:27 smithi089 bash[72306]: ts=2024-07-15T06:55:27.628Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-07-15T06:55:27.872 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 06:55:27 smithi089 bash[72306]: ts=2024-07-15T06:55:27.658Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-07-15T06:55:27.872 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 06:55:27 smithi089 bash[72306]: ts=2024-07-15T06:55:27.659Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-07-15T06:55:27.872 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 06:55:27 smithi089 bash[72306]: ts=2024-07-15T06:55:27.662Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-07-15T06:55:27.872 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 06:55:27 smithi089 bash[72306]: ts=2024-07-15T06:55:27.662Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-07-15T06:55:29.803 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:29 smithi134 bash[24068]: cluster 2024-07-15T06:55:28.526570+0000 mgr.x (mgr.24544) 38 : cluster [DBG] pgmap v17: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:55:29.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:29 smithi134 bash[24068]: audit 2024-07-15T06:55:29.256132+0000 mon.a (mon.0) 1035 : audit [DBG] from='client.? 172.21.15.89:0/2861006145' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:55:29.869 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 06:55:29 smithi089 bash[72306]: ts=2024-07-15T06:55:29.629Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000416854s 2024-07-15T06:55:29.869 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:29 smithi089 bash[21793]: cluster 2024-07-15T06:55:28.526570+0000 mgr.x (mgr.24544) 38 : cluster [DBG] pgmap v17: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:55:29.869 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:29 smithi089 bash[21793]: audit 2024-07-15T06:55:29.256132+0000 mon.a (mon.0) 1035 : audit [DBG] from='client.? 172.21.15.89:0/2861006145' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:55:29.870 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:29 smithi089 bash[31372]: cluster 2024-07-15T06:55:28.526570+0000 mgr.x (mgr.24544) 38 : cluster [DBG] pgmap v17: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:55:29.870 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:29 smithi089 bash[31372]: audit 2024-07-15T06:55:29.256132+0000 mon.a (mon.0) 1035 : audit [DBG] from='client.? 172.21.15.89:0/2861006145' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:55:31.161 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:55:30 smithi134 systemd[1]: Stopping Ceph prometheus.a for 75ba02bc-4274-11ef-bcad-c7b262605968... 2024-07-15T06:55:31.161 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:55:30 smithi134 bash[50757]: ts=2024-07-15T06:55:30.946Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-07-15T06:55:31.161 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:55:30 smithi134 bash[50757]: ts=2024-07-15T06:55:30.946Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-07-15T06:55:31.161 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:55:30 smithi134 bash[50757]: ts=2024-07-15T06:55:30.946Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-07-15T06:55:31.162 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:55:30 smithi134 bash[50757]: ts=2024-07-15T06:55:30.946Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-07-15T06:55:31.162 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:55:30 smithi134 bash[50757]: ts=2024-07-15T06:55:30.946Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-07-15T06:55:31.162 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:55:30 smithi134 bash[50757]: ts=2024-07-15T06:55:30.946Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-07-15T06:55:31.162 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:55:30 smithi134 bash[50757]: ts=2024-07-15T06:55:30.946Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-07-15T06:55:31.162 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:55:30 smithi134 bash[50757]: ts=2024-07-15T06:55:30.946Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-07-15T06:55:31.162 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:55:30 smithi134 bash[50757]: ts=2024-07-15T06:55:30.946Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-07-15T06:55:31.162 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:55:30 smithi134 bash[50757]: ts=2024-07-15T06:55:30.947Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-07-15T06:55:31.162 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:55:30 smithi134 bash[50757]: ts=2024-07-15T06:55:30.947Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-07-15T06:55:31.162 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:55:30 smithi134 bash[50757]: ts=2024-07-15T06:55:30.947Z caller=main.go:1147 level=info msg="See you next time!" 2024-07-15T06:55:31.724 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:55:31 smithi134 bash[58867]: ceph-75ba02bc-4274-11ef-bcad-c7b262605968-prometheus-a 2024-07-15T06:55:31.724 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:55:31 smithi134 bash[58925]: Error response from daemon: No such container: ceph-75ba02bc-4274-11ef-bcad-c7b262605968-prometheus-a 2024-07-15T06:55:31.725 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:55:31 smithi134 systemd[1]: ceph-75ba02bc-4274-11ef-bcad-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-07-15T06:55:31.725 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:55:31 smithi134 systemd[1]: Stopped Ceph prometheus.a for 75ba02bc-4274-11ef-bcad-c7b262605968. 2024-07-15T06:55:31.725 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:55:31 smithi134 systemd[1]: Started Ceph prometheus.a for 75ba02bc-4274-11ef-bcad-c7b262605968. 2024-07-15T06:55:31.725 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:31 smithi134 bash[24068]: cluster 2024-07-15T06:55:30.527219+0000 mgr.x (mgr.24544) 39 : cluster [DBG] pgmap v18: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:55:31.725 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:31 smithi134 bash[24068]: audit 2024-07-15T06:55:31.080058+0000 mon.b (mon.1) 67 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:55:31.869 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:31 smithi089 bash[21793]: cluster 2024-07-15T06:55:30.527219+0000 mgr.x (mgr.24544) 39 : cluster [DBG] pgmap v18: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:55:31.869 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:31 smithi089 bash[21793]: audit 2024-07-15T06:55:31.080058+0000 mon.b (mon.1) 67 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:55:31.869 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:31 smithi089 bash[31372]: cluster 2024-07-15T06:55:30.527219+0000 mgr.x (mgr.24544) 39 : cluster [DBG] pgmap v18: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:55:31.869 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:31 smithi089 bash[31372]: audit 2024-07-15T06:55:31.080058+0000 mon.b (mon.1) 67 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:55:32.053 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:55:31 smithi134 systemd[1]: Stopping Ceph node-exporter.b for 75ba02bc-4274-11ef-bcad-c7b262605968... 2024-07-15T06:55:32.869 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:32 smithi089 bash[21793]: audit 2024-07-15T06:55:31.627159+0000 mon.c (mon.2) 48 : audit [DBG] from='client.? 172.21.15.89:0/4192290158' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:55:32.869 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:32 smithi089 bash[21793]: audit 2024-07-15T06:55:31.665489+0000 mon.a (mon.0) 1036 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:32.869 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:32 smithi089 bash[21793]: audit 2024-07-15T06:55:31.674520+0000 mon.a (mon.0) 1037 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:32.869 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:32 smithi089 bash[21793]: cephadm 2024-07-15T06:55:31.676494+0000 mgr.x (mgr.24544) 40 : cephadm [INF] Reconfiguring node-exporter.b (dependencies changed)... 2024-07-15T06:55:32.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:32 smithi089 bash[21793]: cephadm 2024-07-15T06:55:31.676808+0000 mgr.x (mgr.24544) 41 : cephadm [INF] Reconfiguring daemon node-exporter.b on smithi134 2024-07-15T06:55:32.870 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:32 smithi089 bash[31372]: audit 2024-07-15T06:55:31.627159+0000 mon.c (mon.2) 48 : audit [DBG] from='client.? 172.21.15.89:0/4192290158' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:55:32.870 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:32 smithi089 bash[31372]: audit 2024-07-15T06:55:31.665489+0000 mon.a (mon.0) 1036 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:32.870 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:32 smithi089 bash[31372]: audit 2024-07-15T06:55:31.674520+0000 mon.a (mon.0) 1037 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:32.870 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:32 smithi089 bash[31372]: cephadm 2024-07-15T06:55:31.676494+0000 mgr.x (mgr.24544) 40 : cephadm [INF] Reconfiguring node-exporter.b (dependencies changed)... 2024-07-15T06:55:32.870 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:32 smithi089 bash[31372]: cephadm 2024-07-15T06:55:31.676808+0000 mgr.x (mgr.24544) 41 : cephadm [INF] Reconfiguring daemon node-exporter.b on smithi134 2024-07-15T06:55:33.053 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:32 smithi134 bash[24068]: audit 2024-07-15T06:55:31.627159+0000 mon.c (mon.2) 48 : audit [DBG] from='client.? 172.21.15.89:0/4192290158' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:55:33.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:32 smithi134 bash[24068]: audit 2024-07-15T06:55:31.665489+0000 mon.a (mon.0) 1036 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:33.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:32 smithi134 bash[24068]: audit 2024-07-15T06:55:31.674520+0000 mon.a (mon.0) 1037 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:33.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:32 smithi134 bash[24068]: cephadm 2024-07-15T06:55:31.676494+0000 mgr.x (mgr.24544) 40 : cephadm [INF] Reconfiguring node-exporter.b (dependencies changed)... 2024-07-15T06:55:33.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:32 smithi134 bash[24068]: cephadm 2024-07-15T06:55:31.676808+0000 mgr.x (mgr.24544) 41 : cephadm [INF] Reconfiguring daemon node-exporter.b on smithi134 2024-07-15T06:55:33.493 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:55:33 smithi134 bash[59001]: ceph-75ba02bc-4274-11ef-bcad-c7b262605968-node-exporter-b 2024-07-15T06:55:33.749 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:33 smithi134 bash[24068]: cluster 2024-07-15T06:55:32.528378+0000 mgr.x (mgr.24544) 42 : cluster [DBG] pgmap v19: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:55:33.750 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:33 smithi134 bash[24068]: audit 2024-07-15T06:55:33.557258+0000 mon.a (mon.0) 1038 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:33.750 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:33 smithi134 bash[24068]: audit 2024-07-15T06:55:33.567331+0000 mon.a (mon.0) 1039 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:33.750 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:55:33 smithi134 systemd[1]: ceph-75ba02bc-4274-11ef-bcad-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-07-15T06:55:33.750 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:55:33 smithi134 systemd[1]: ceph-75ba02bc-4274-11ef-bcad-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-07-15T06:55:33.750 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:55:33 smithi134 systemd[1]: Stopped Ceph node-exporter.b for 75ba02bc-4274-11ef-bcad-c7b262605968. 2024-07-15T06:55:33.750 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:55:33 smithi134 systemd[1]: Started Ceph node-exporter.b for 75ba02bc-4274-11ef-bcad-c7b262605968. 2024-07-15T06:55:34.054 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:55:33 smithi134 bash[58954]: ts=2024-07-15T06:55:33.755Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-07-15T06:55:34.054 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:55:33 smithi134 bash[58954]: ts=2024-07-15T06:55:33.755Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-07-15T06:55:34.054 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:55:33 smithi134 bash[58954]: ts=2024-07-15T06:55:33.756Z caller=main.go:570 level=info host_details="(Linux 5.15.0-113-generic #123-Ubuntu SMP Mon Jun 10 08:16:17 UTC 2024 x86_64 smithi134 (none))" 2024-07-15T06:55:34.054 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:55:33 smithi134 bash[58954]: ts=2024-07-15T06:55:33.756Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-07-15T06:55:34.054 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:55:33 smithi134 bash[58954]: ts=2024-07-15T06:55:33.756Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-07-15T06:55:34.054 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:55:33 smithi134 bash[58954]: ts=2024-07-15T06:55:33.757Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-07-15T06:55:34.055 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:55:33 smithi134 bash[58954]: ts=2024-07-15T06:55:33.759Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-07-15T06:55:34.055 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:55:33 smithi134 bash[58954]: ts=2024-07-15T06:55:33.760Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-07-15T06:55:34.055 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:55:33 smithi134 bash[58954]: ts=2024-07-15T06:55:33.760Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-07-15T06:55:34.055 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:55:33 smithi134 bash[58954]: ts=2024-07-15T06:55:33.761Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-07-15T06:55:34.055 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:55:33 smithi134 bash[58954]: ts=2024-07-15T06:55:33.761Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.703µs 2024-07-15T06:55:34.055 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:55:33 smithi134 bash[58954]: ts=2024-07-15T06:55:33.761Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-07-15T06:55:34.055 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:55:33 smithi134 bash[58954]: ts=2024-07-15T06:55:33.762Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=2 2024-07-15T06:55:34.055 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:55:33 smithi134 bash[58954]: ts=2024-07-15T06:55:33.786Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=2 2024-07-15T06:55:34.055 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:55:33 smithi134 bash[58954]: ts=2024-07-15T06:55:33.786Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=2 2024-07-15T06:55:34.055 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:55:33 smithi134 bash[58954]: ts=2024-07-15T06:55:33.786Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=35.004µs wal_replay_duration=24.638871ms wbl_replay_duration=113ns total_replay_duration=24.702882ms 2024-07-15T06:55:34.055 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:55:33 smithi134 bash[58954]: ts=2024-07-15T06:55:33.789Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-07-15T06:55:34.055 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:55:33 smithi134 bash[58954]: ts=2024-07-15T06:55:33.789Z caller=main.go:1029 level=info msg="TSDB started" 2024-07-15T06:55:34.056 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:55:33 smithi134 bash[58954]: ts=2024-07-15T06:55:33.789Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-07-15T06:55:34.056 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:55:33 smithi134 bash[58954]: ts=2024-07-15T06:55:33.812Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=23.000117ms db_storage=1.015µs remote_storage=1.285µs web_handler=377ns query_engine=984ns scrape=263.471µs scrape_sd=113.936µs notify=11.972µs notify_sd=27.687µs rules=22.141873ms tracing=6.526µs 2024-07-15T06:55:34.056 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:55:33 smithi134 bash[58954]: ts=2024-07-15T06:55:33.812Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-07-15T06:55:34.056 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:55:33 smithi134 bash[58954]: ts=2024-07-15T06:55:33.812Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-07-15T06:55:34.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:33 smithi089 bash[21793]: cluster 2024-07-15T06:55:32.528378+0000 mgr.x (mgr.24544) 42 : cluster [DBG] pgmap v19: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:55:34.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:33 smithi089 bash[21793]: audit 2024-07-15T06:55:33.557258+0000 mon.a (mon.0) 1038 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:34.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:33 smithi089 bash[21793]: audit 2024-07-15T06:55:33.567331+0000 mon.a (mon.0) 1039 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:34.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:33 smithi089 bash[31372]: cluster 2024-07-15T06:55:32.528378+0000 mgr.x (mgr.24544) 42 : cluster [DBG] pgmap v19: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:55:34.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:33 smithi089 bash[31372]: audit 2024-07-15T06:55:33.557258+0000 mon.a (mon.0) 1038 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:34.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:33 smithi089 bash[31372]: audit 2024-07-15T06:55:33.567331+0000 mon.a (mon.0) 1039 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:35.053 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:34 smithi134 bash[24068]: cephadm 2024-07-15T06:55:33.569772+0000 mgr.x (mgr.24544) 43 : cephadm [INF] Reconfiguring grafana.a (dependencies changed)... 2024-07-15T06:55:35.053 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:34 smithi134 bash[24068]: cephadm 2024-07-15T06:55:33.618008+0000 mgr.x (mgr.24544) 44 : cephadm [INF] Reconfiguring daemon grafana.a on smithi134 2024-07-15T06:55:35.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:34 smithi134 bash[24068]: audit 2024-07-15T06:55:33.998120+0000 mon.c (mon.2) 49 : audit [DBG] from='client.? 172.21.15.89:0/1917354954' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:55:35.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:34 smithi089 bash[21793]: cephadm 2024-07-15T06:55:33.569772+0000 mgr.x (mgr.24544) 43 : cephadm [INF] Reconfiguring grafana.a (dependencies changed)... 2024-07-15T06:55:35.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:34 smithi089 bash[21793]: cephadm 2024-07-15T06:55:33.618008+0000 mgr.x (mgr.24544) 44 : cephadm [INF] Reconfiguring daemon grafana.a on smithi134 2024-07-15T06:55:35.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:34 smithi089 bash[21793]: audit 2024-07-15T06:55:33.998120+0000 mon.c (mon.2) 49 : audit [DBG] from='client.? 172.21.15.89:0/1917354954' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:55:35.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:34 smithi089 bash[31372]: cephadm 2024-07-15T06:55:33.569772+0000 mgr.x (mgr.24544) 43 : cephadm [INF] Reconfiguring grafana.a (dependencies changed)... 2024-07-15T06:55:35.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:34 smithi089 bash[31372]: cephadm 2024-07-15T06:55:33.618008+0000 mgr.x (mgr.24544) 44 : cephadm [INF] Reconfiguring daemon grafana.a on smithi134 2024-07-15T06:55:35.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:34 smithi089 bash[31372]: audit 2024-07-15T06:55:33.998120+0000 mon.c (mon.2) 49 : audit [DBG] from='client.? 172.21.15.89:0/1917354954' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:55:35.901 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:35 smithi134 bash[24068]: cluster 2024-07-15T06:55:34.529043+0000 mgr.x (mgr.24544) 45 : cluster [DBG] pgmap v20: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:55:35.902 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:55:35 smithi134 bash[59088]: ts=2024-07-15T06:55:35.740Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-07-15T06:55:35.902 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:55:35 smithi134 bash[59088]: ts=2024-07-15T06:55:35.740Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-07-15T06:55:35.902 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:55:35 smithi134 bash[59088]: ts=2024-07-15T06:55:35.741Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-07-15T06:55:35.902 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:55:35 smithi134 bash[59088]: ts=2024-07-15T06:55:35.741Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-07-15T06:55:35.902 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:55:35 smithi134 bash[59088]: ts=2024-07-15T06:55:35.741Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-07-15T06:55:35.902 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:55:35 smithi134 bash[59088]: ts=2024-07-15T06:55:35.741Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-07-15T06:55:35.902 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:55:35 smithi134 bash[59088]: ts=2024-07-15T06:55:35.742Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-07-15T06:55:35.902 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:55:35 smithi134 bash[59088]: ts=2024-07-15T06:55:35.742Z caller=node_exporter.go:117 level=info collector=arp 2024-07-15T06:55:35.903 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:55:35 smithi134 bash[59088]: ts=2024-07-15T06:55:35.742Z caller=node_exporter.go:117 level=info collector=bcache 2024-07-15T06:55:35.903 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:55:35 smithi134 bash[59088]: ts=2024-07-15T06:55:35.742Z caller=node_exporter.go:117 level=info collector=bonding 2024-07-15T06:55:35.903 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:55:35 smithi134 bash[59088]: ts=2024-07-15T06:55:35.742Z caller=node_exporter.go:117 level=info collector=btrfs 2024-07-15T06:55:35.903 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:55:35 smithi134 bash[59088]: ts=2024-07-15T06:55:35.742Z caller=node_exporter.go:117 level=info collector=conntrack 2024-07-15T06:55:35.903 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:55:35 smithi134 bash[59088]: ts=2024-07-15T06:55:35.742Z caller=node_exporter.go:117 level=info collector=cpu 2024-07-15T06:55:35.903 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:55:35 smithi134 bash[59088]: ts=2024-07-15T06:55:35.742Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-07-15T06:55:35.903 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:55:35 smithi134 bash[59088]: ts=2024-07-15T06:55:35.742Z caller=node_exporter.go:117 level=info collector=diskstats 2024-07-15T06:55:35.903 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:55:35 smithi134 bash[59088]: ts=2024-07-15T06:55:35.742Z caller=node_exporter.go:117 level=info collector=dmi 2024-07-15T06:55:35.903 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:55:35 smithi134 bash[59088]: ts=2024-07-15T06:55:35.742Z caller=node_exporter.go:117 level=info collector=edac 2024-07-15T06:55:35.903 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:55:35 smithi134 bash[59088]: ts=2024-07-15T06:55:35.742Z caller=node_exporter.go:117 level=info collector=entropy 2024-07-15T06:55:35.903 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:55:35 smithi134 bash[59088]: ts=2024-07-15T06:55:35.742Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-07-15T06:55:35.903 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:55:35 smithi134 bash[59088]: ts=2024-07-15T06:55:35.742Z caller=node_exporter.go:117 level=info collector=filefd 2024-07-15T06:55:35.903 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:55:35 smithi134 bash[59088]: ts=2024-07-15T06:55:35.742Z caller=node_exporter.go:117 level=info collector=filesystem 2024-07-15T06:55:35.904 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:55:35 smithi134 bash[59088]: ts=2024-07-15T06:55:35.742Z caller=node_exporter.go:117 level=info collector=hwmon 2024-07-15T06:55:35.904 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:55:35 smithi134 bash[59088]: ts=2024-07-15T06:55:35.742Z caller=node_exporter.go:117 level=info collector=infiniband 2024-07-15T06:55:35.904 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:55:35 smithi134 bash[59088]: ts=2024-07-15T06:55:35.742Z caller=node_exporter.go:117 level=info collector=ipvs 2024-07-15T06:55:35.904 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:55:35 smithi134 bash[59088]: ts=2024-07-15T06:55:35.742Z caller=node_exporter.go:117 level=info collector=loadavg 2024-07-15T06:55:35.904 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:55:35 smithi134 bash[59088]: ts=2024-07-15T06:55:35.742Z caller=node_exporter.go:117 level=info collector=mdadm 2024-07-15T06:55:35.904 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:55:35 smithi134 bash[59088]: ts=2024-07-15T06:55:35.742Z caller=node_exporter.go:117 level=info collector=meminfo 2024-07-15T06:55:35.904 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:55:35 smithi134 bash[59088]: ts=2024-07-15T06:55:35.742Z caller=node_exporter.go:117 level=info collector=netclass 2024-07-15T06:55:35.904 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:55:35 smithi134 bash[59088]: ts=2024-07-15T06:55:35.742Z caller=node_exporter.go:117 level=info collector=netdev 2024-07-15T06:55:35.904 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:55:35 smithi134 bash[59088]: ts=2024-07-15T06:55:35.742Z caller=node_exporter.go:117 level=info collector=netstat 2024-07-15T06:55:35.904 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:55:35 smithi134 bash[59088]: ts=2024-07-15T06:55:35.742Z caller=node_exporter.go:117 level=info collector=nfs 2024-07-15T06:55:35.904 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:55:35 smithi134 bash[59088]: ts=2024-07-15T06:55:35.742Z caller=node_exporter.go:117 level=info collector=nfsd 2024-07-15T06:55:35.904 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:55:35 smithi134 bash[59088]: ts=2024-07-15T06:55:35.742Z caller=node_exporter.go:117 level=info collector=nvme 2024-07-15T06:55:35.904 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:55:35 smithi134 bash[59088]: ts=2024-07-15T06:55:35.742Z caller=node_exporter.go:117 level=info collector=os 2024-07-15T06:55:35.904 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:55:35 smithi134 bash[59088]: ts=2024-07-15T06:55:35.742Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-07-15T06:55:35.905 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:55:35 smithi134 bash[59088]: ts=2024-07-15T06:55:35.742Z caller=node_exporter.go:117 level=info collector=pressure 2024-07-15T06:55:35.905 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:55:35 smithi134 bash[59088]: ts=2024-07-15T06:55:35.742Z caller=node_exporter.go:117 level=info collector=rapl 2024-07-15T06:55:35.905 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:55:35 smithi134 bash[59088]: ts=2024-07-15T06:55:35.742Z caller=node_exporter.go:117 level=info collector=schedstat 2024-07-15T06:55:35.905 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:55:35 smithi134 bash[59088]: ts=2024-07-15T06:55:35.742Z caller=node_exporter.go:117 level=info collector=selinux 2024-07-15T06:55:35.905 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:55:35 smithi134 bash[59088]: ts=2024-07-15T06:55:35.742Z caller=node_exporter.go:117 level=info collector=sockstat 2024-07-15T06:55:35.905 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:55:35 smithi134 bash[59088]: ts=2024-07-15T06:55:35.742Z caller=node_exporter.go:117 level=info collector=softnet 2024-07-15T06:55:35.905 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:55:35 smithi134 bash[59088]: ts=2024-07-15T06:55:35.742Z caller=node_exporter.go:117 level=info collector=stat 2024-07-15T06:55:35.905 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:55:35 smithi134 bash[59088]: ts=2024-07-15T06:55:35.742Z caller=node_exporter.go:117 level=info collector=tapestats 2024-07-15T06:55:35.905 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:55:35 smithi134 bash[59088]: ts=2024-07-15T06:55:35.742Z caller=node_exporter.go:117 level=info collector=textfile 2024-07-15T06:55:35.905 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:55:35 smithi134 bash[59088]: ts=2024-07-15T06:55:35.742Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-07-15T06:55:35.905 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:55:35 smithi134 bash[59088]: ts=2024-07-15T06:55:35.742Z caller=node_exporter.go:117 level=info collector=time 2024-07-15T06:55:35.906 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:55:35 smithi134 bash[59088]: ts=2024-07-15T06:55:35.742Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-07-15T06:55:35.906 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:55:35 smithi134 bash[59088]: ts=2024-07-15T06:55:35.742Z caller=node_exporter.go:117 level=info collector=uname 2024-07-15T06:55:35.906 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:55:35 smithi134 bash[59088]: ts=2024-07-15T06:55:35.742Z caller=node_exporter.go:117 level=info collector=vmstat 2024-07-15T06:55:35.906 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:55:35 smithi134 bash[59088]: ts=2024-07-15T06:55:35.742Z caller=node_exporter.go:117 level=info collector=xfs 2024-07-15T06:55:35.906 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:55:35 smithi134 bash[59088]: ts=2024-07-15T06:55:35.742Z caller=node_exporter.go:117 level=info collector=zfs 2024-07-15T06:55:35.906 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:55:35 smithi134 bash[59088]: ts=2024-07-15T06:55:35.742Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-07-15T06:55:35.906 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:55:35 smithi134 bash[59088]: ts=2024-07-15T06:55:35.743Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-07-15T06:55:36.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:35 smithi089 bash[21793]: cluster 2024-07-15T06:55:34.529043+0000 mgr.x (mgr.24544) 45 : cluster [DBG] pgmap v20: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:55:36.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:35 smithi089 bash[31372]: cluster 2024-07-15T06:55:34.529043+0000 mgr.x (mgr.24544) 45 : cluster [DBG] pgmap v20: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:55:37.053 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:36 smithi134 bash[24068]: audit 2024-07-15T06:55:36.367338+0000 mon.c (mon.2) 50 : audit [DBG] from='client.? 172.21.15.89:0/484553734' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:55:37.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:36 smithi089 bash[21793]: audit 2024-07-15T06:55:36.367338+0000 mon.c (mon.2) 50 : audit [DBG] from='client.? 172.21.15.89:0/484553734' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:55:37.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:36 smithi089 bash[31372]: audit 2024-07-15T06:55:36.367338+0000 mon.c (mon.2) 50 : audit [DBG] from='client.? 172.21.15.89:0/484553734' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:55:38.053 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:37 smithi134 bash[24068]: cluster 2024-07-15T06:55:36.529686+0000 mgr.x (mgr.24544) 46 : cluster [DBG] pgmap v21: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:55:38.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:37 smithi089 bash[21793]: cluster 2024-07-15T06:55:36.529686+0000 mgr.x (mgr.24544) 46 : cluster [DBG] pgmap v21: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:55:38.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:37 smithi089 bash[31372]: cluster 2024-07-15T06:55:36.529686+0000 mgr.x (mgr.24544) 46 : cluster [DBG] pgmap v21: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:55:38.120 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 06:55:37 smithi089 bash[72306]: ts=2024-07-15T06:55:37.630Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.001696897s 2024-07-15T06:55:38.805 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:55:38 smithi134 systemd[1]: Stopping Ceph grafana.a for 75ba02bc-4274-11ef-bcad-c7b262605968... 2024-07-15T06:55:39.553 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:55:39 smithi134 bash[59376]: ceph-75ba02bc-4274-11ef-bcad-c7b262605968-grafana-a 2024-07-15T06:55:39.554 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:55:39 smithi134 systemd[1]: ceph-75ba02bc-4274-11ef-bcad-c7b262605968@grafana.a.service: Main process exited, code=exited, status=143/n/a 2024-07-15T06:55:39.554 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:55:39 smithi134 bash[59435]: Error response from daemon: No such container: ceph-75ba02bc-4274-11ef-bcad-c7b262605968-grafana-a 2024-07-15T06:55:39.554 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:55:39 smithi134 systemd[1]: ceph-75ba02bc-4274-11ef-bcad-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-07-15T06:55:39.554 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:55:39 smithi134 systemd[1]: Stopped Ceph grafana.a for 75ba02bc-4274-11ef-bcad-c7b262605968. 2024-07-15T06:55:39.554 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:55:39 smithi134 systemd[1]: Started Ceph grafana.a for 75ba02bc-4274-11ef-bcad-c7b262605968. 2024-07-15T06:55:40.053 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:55:39 smithi134 bash[52600]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:55:39] "GET /metrics HTTP/1.1" 200 33802 "" "Prometheus/2.43.0" 2024-07-15T06:55:40.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:39 smithi134 bash[24068]: cluster 2024-07-15T06:55:38.530566+0000 mgr.x (mgr.24544) 47 : cluster [DBG] pgmap v22: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:55:40.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:39 smithi134 bash[24068]: audit 2024-07-15T06:55:38.725301+0000 mon.c (mon.2) 51 : audit [DBG] from='client.? 172.21.15.89:0/979434718' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:55:40.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:39 smithi134 bash[24068]: audit 2024-07-15T06:55:39.341895+0000 mon.a (mon.0) 1040 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:40.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:39 smithi134 bash[24068]: audit 2024-07-15T06:55:39.351556+0000 mon.a (mon.0) 1041 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:40.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:39 smithi134 bash[24068]: audit 2024-07-15T06:55:39.357156+0000 mon.b (mon.1) 68 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-07-15T06:55:40.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:39 smithi134 bash[24068]: audit 2024-07-15T06:55:39.361002+0000 mon.b (mon.1) 69 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-07-15T06:55:40.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:39 smithi134 bash[24068]: audit 2024-07-15T06:55:39.364202+0000 mon.b (mon.1) 70 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-07-15T06:55:40.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:39 smithi134 bash[24068]: audit 2024-07-15T06:55:39.404862+0000 mon.b (mon.1) 71 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:55:40.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:39 smithi089 bash[21793]: cluster 2024-07-15T06:55:38.530566+0000 mgr.x (mgr.24544) 47 : cluster [DBG] pgmap v22: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:55:40.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:39 smithi089 bash[21793]: audit 2024-07-15T06:55:38.725301+0000 mon.c (mon.2) 51 : audit [DBG] from='client.? 172.21.15.89:0/979434718' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:55:40.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:39 smithi089 bash[21793]: audit 2024-07-15T06:55:39.341895+0000 mon.a (mon.0) 1040 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:40.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:39 smithi089 bash[21793]: audit 2024-07-15T06:55:39.351556+0000 mon.a (mon.0) 1041 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:40.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:39 smithi089 bash[21793]: audit 2024-07-15T06:55:39.357156+0000 mon.b (mon.1) 68 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-07-15T06:55:40.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:39 smithi089 bash[21793]: audit 2024-07-15T06:55:39.361002+0000 mon.b (mon.1) 69 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-07-15T06:55:40.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:39 smithi089 bash[21793]: audit 2024-07-15T06:55:39.364202+0000 mon.b (mon.1) 70 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-07-15T06:55:40.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:39 smithi089 bash[21793]: audit 2024-07-15T06:55:39.404862+0000 mon.b (mon.1) 71 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:55:40.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:39 smithi089 bash[31372]: cluster 2024-07-15T06:55:38.530566+0000 mgr.x (mgr.24544) 47 : cluster [DBG] pgmap v22: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:55:40.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:39 smithi089 bash[31372]: audit 2024-07-15T06:55:38.725301+0000 mon.c (mon.2) 51 : audit [DBG] from='client.? 172.21.15.89:0/979434718' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:55:40.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:39 smithi089 bash[31372]: audit 2024-07-15T06:55:39.341895+0000 mon.a (mon.0) 1040 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:40.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:39 smithi089 bash[31372]: audit 2024-07-15T06:55:39.351556+0000 mon.a (mon.0) 1041 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:40.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:39 smithi089 bash[31372]: audit 2024-07-15T06:55:39.357156+0000 mon.b (mon.1) 68 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-07-15T06:55:40.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:39 smithi089 bash[31372]: audit 2024-07-15T06:55:39.361002+0000 mon.b (mon.1) 69 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-07-15T06:55:40.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:39 smithi089 bash[31372]: audit 2024-07-15T06:55:39.364202+0000 mon.b (mon.1) 70 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-07-15T06:55:40.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:39 smithi089 bash[31372]: audit 2024-07-15T06:55:39.404862+0000 mon.b (mon.1) 71 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:55:40.725 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:40 smithi134 bash[24068]: audit 2024-07-15T06:55:39.357593+0000 mgr.x (mgr.24544) 48 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-07-15T06:55:40.725 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:40 smithi134 bash[24068]: audit 2024-07-15T06:55:39.361381+0000 mgr.x (mgr.24544) 49 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-07-15T06:55:40.725 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:40 smithi134 bash[24068]: audit 2024-07-15T06:55:39.364511+0000 mgr.x (mgr.24544) 50 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-07-15T06:55:40.725 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:40 smithi134 bash[24068]: cephadm 2024-07-15T06:55:39.407794+0000 mgr.x (mgr.24544) 51 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2024-07-15T06:55:40.726 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:40 smithi134 bash[24068]: cephadm 2024-07-15T06:55:39.408276+0000 mgr.x (mgr.24544) 52 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2024-07-15T06:55:40.726 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:55:40 smithi134 bash[59462]: logger=settings t=2024-07-15T06:55:40.707848805Z level=info msg="Starting Grafana" version=9.4.7 commit=4add91f03d branch=HEAD compiled=2023-03-16T23:56:52Z 2024-07-15T06:55:40.726 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:55:40 smithi134 bash[59462]: logger=settings t=2024-07-15T06:55:40.708044058Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2024-07-15T06:55:40.726 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:55:40 smithi134 bash[59462]: logger=settings t=2024-07-15T06:55:40.708080445Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2024-07-15T06:55:40.726 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:55:40 smithi134 bash[59462]: logger=settings t=2024-07-15T06:55:40.708099947Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2024-07-15T06:55:40.726 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:55:40 smithi134 bash[59462]: logger=settings t=2024-07-15T06:55:40.708113989Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2024-07-15T06:55:40.726 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:55:40 smithi134 bash[59462]: logger=settings t=2024-07-15T06:55:40.708129397Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2024-07-15T06:55:40.726 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:55:40 smithi134 bash[59462]: logger=settings t=2024-07-15T06:55:40.708141885Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-07-15T06:55:40.727 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:55:40 smithi134 bash[59462]: logger=settings t=2024-07-15T06:55:40.708149154Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-07-15T06:55:40.727 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:55:40 smithi134 bash[59462]: logger=settings t=2024-07-15T06:55:40.708156322Z level=info msg="Path Home" path=/usr/share/grafana 2024-07-15T06:55:40.727 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:55:40 smithi134 bash[59462]: logger=settings t=2024-07-15T06:55:40.708168587Z level=info msg="Path Data" path=/var/lib/grafana 2024-07-15T06:55:40.727 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:55:40 smithi134 bash[59462]: logger=settings t=2024-07-15T06:55:40.708179301Z level=info msg="Path Logs" path=/var/log/grafana 2024-07-15T06:55:40.727 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:55:40 smithi134 bash[59462]: logger=settings t=2024-07-15T06:55:40.708196322Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2024-07-15T06:55:40.727 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:55:40 smithi134 bash[59462]: logger=settings t=2024-07-15T06:55:40.708215661Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2024-07-15T06:55:40.727 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:55:40 smithi134 bash[59462]: logger=settings t=2024-07-15T06:55:40.708226569Z level=info msg="App mode production" 2024-07-15T06:55:40.727 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:55:40 smithi134 bash[59462]: logger=sqlstore t=2024-07-15T06:55:40.708302747Z level=info msg="Connecting to DB" dbtype=sqlite3 2024-07-15T06:55:40.727 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:55:40 smithi134 bash[59462]: logger=sqlstore t=2024-07-15T06:55:40.708332069Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-07-15T06:55:40.727 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:55:40 smithi134 bash[59462]: logger=migrator t=2024-07-15T06:55:40.726340214Z level=info msg="Starting DB migrations" 2024-07-15T06:55:40.727 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:55:40 smithi134 bash[59462]: logger=migrator t=2024-07-15T06:55:40.731861676Z level=info msg="migrations completed" performed=0 skipped=481 duration=515.411µs 2024-07-15T06:55:40.727 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:55:40 smithi134 bash[59462]: logger=sqlstore t=2024-07-15T06:55:40.732388452Z level=info msg="Created default organization" 2024-07-15T06:55:40.966 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:40 smithi089 bash[21793]: audit 2024-07-15T06:55:39.357593+0000 mgr.x (mgr.24544) 48 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-07-15T06:55:40.966 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:40 smithi089 bash[21793]: audit 2024-07-15T06:55:39.361381+0000 mgr.x (mgr.24544) 49 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-07-15T06:55:40.966 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:40 smithi089 bash[21793]: audit 2024-07-15T06:55:39.364511+0000 mgr.x (mgr.24544) 50 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-07-15T06:55:40.967 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:40 smithi089 bash[21793]: cephadm 2024-07-15T06:55:39.407794+0000 mgr.x (mgr.24544) 51 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2024-07-15T06:55:40.967 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:40 smithi089 bash[21793]: cephadm 2024-07-15T06:55:39.408276+0000 mgr.x (mgr.24544) 52 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2024-07-15T06:55:40.967 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:40 smithi089 bash[31372]: audit 2024-07-15T06:55:39.357593+0000 mgr.x (mgr.24544) 48 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-07-15T06:55:40.967 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:40 smithi089 bash[31372]: audit 2024-07-15T06:55:39.361381+0000 mgr.x (mgr.24544) 49 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-07-15T06:55:40.967 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:40 smithi089 bash[31372]: audit 2024-07-15T06:55:39.364511+0000 mgr.x (mgr.24544) 50 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-07-15T06:55:40.967 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:40 smithi089 bash[31372]: cephadm 2024-07-15T06:55:39.407794+0000 mgr.x (mgr.24544) 51 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2024-07-15T06:55:40.967 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:40 smithi089 bash[31372]: cephadm 2024-07-15T06:55:39.408276+0000 mgr.x (mgr.24544) 52 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2024-07-15T06:55:41.053 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:55:40 smithi134 bash[59462]: logger=plugin.loader t=2024-07-15T06:55:40.825209012Z level=info msg="Plugin registered" pluginID=input 2024-07-15T06:55:41.054 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:55:40 smithi134 bash[59462]: logger=plugin.loader t=2024-07-15T06:55:40.855516492Z level=info msg="Plugin registered" pluginID=grafana-piechart-panel 2024-07-15T06:55:41.054 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:55:40 smithi134 bash[59462]: logger=plugin.loader t=2024-07-15T06:55:40.855563639Z level=info msg="Plugin registered" pluginID=vonage-status-panel 2024-07-15T06:55:41.054 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:55:40 smithi134 bash[59462]: logger=secrets t=2024-07-15T06:55:40.855611731Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2024-07-15T06:55:41.054 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:55:40 smithi134 bash[59462]: logger=query_data t=2024-07-15T06:55:40.85758375Z level=info msg="Query Service initialization" 2024-07-15T06:55:41.054 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:55:40 smithi134 bash[59462]: logger=live.push_http t=2024-07-15T06:55:40.861852868Z level=info msg="Live Push Gateway initialization" 2024-07-15T06:55:42.006 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:41 smithi089 bash[21793]: cluster 2024-07-15T06:55:40.530971+0000 mgr.x (mgr.24544) 53 : cluster [DBG] pgmap v23: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:55:42.006 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:41 smithi089 bash[21793]: audit 2024-07-15T06:55:41.091678+0000 mon.a (mon.0) 1042 : audit [DBG] from='client.? 172.21.15.89:0/1353365852' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:55:42.006 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:41 smithi089 bash[21793]: audit 2024-07-15T06:55:41.636271+0000 mon.a (mon.0) 1043 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:42.006 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:41 smithi089 bash[21793]: audit 2024-07-15T06:55:41.639136+0000 mon.b (mon.1) 72 : audit [INF] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-07-15T06:55:42.006 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:41 smithi089 bash[21793]: audit 2024-07-15T06:55:41.639205+0000 mon.a (mon.0) 1044 : audit [INF] from='mgr.24544 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-07-15T06:55:42.006 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:41 smithi089 bash[21793]: audit 2024-07-15T06:55:41.642215+0000 mon.b (mon.1) 73 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-07-15T06:55:42.006 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:41 smithi089 bash[21793]: audit 2024-07-15T06:55:41.644026+0000 mon.b (mon.1) 74 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:55:42.007 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:41 smithi089 bash[31372]: cluster 2024-07-15T06:55:40.530971+0000 mgr.x (mgr.24544) 53 : cluster [DBG] pgmap v23: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:55:42.007 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:41 smithi089 bash[31372]: audit 2024-07-15T06:55:41.091678+0000 mon.a (mon.0) 1042 : audit [DBG] from='client.? 172.21.15.89:0/1353365852' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:55:42.007 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:41 smithi089 bash[31372]: audit 2024-07-15T06:55:41.636271+0000 mon.a (mon.0) 1043 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:42.007 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:41 smithi089 bash[31372]: audit 2024-07-15T06:55:41.639136+0000 mon.b (mon.1) 72 : audit [INF] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-07-15T06:55:42.007 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:41 smithi089 bash[31372]: audit 2024-07-15T06:55:41.639205+0000 mon.a (mon.0) 1044 : audit [INF] from='mgr.24544 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-07-15T06:55:42.007 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:41 smithi089 bash[31372]: audit 2024-07-15T06:55:41.642215+0000 mon.b (mon.1) 73 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-07-15T06:55:42.007 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:41 smithi089 bash[31372]: audit 2024-07-15T06:55:41.644026+0000 mon.b (mon.1) 74 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:55:42.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:41 smithi134 bash[24068]: cluster 2024-07-15T06:55:40.530971+0000 mgr.x (mgr.24544) 53 : cluster [DBG] pgmap v23: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:55:42.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:41 smithi134 bash[24068]: audit 2024-07-15T06:55:41.091678+0000 mon.a (mon.0) 1042 : audit [DBG] from='client.? 172.21.15.89:0/1353365852' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:55:42.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:41 smithi134 bash[24068]: audit 2024-07-15T06:55:41.636271+0000 mon.a (mon.0) 1043 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:42.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:41 smithi134 bash[24068]: audit 2024-07-15T06:55:41.639136+0000 mon.b (mon.1) 72 : audit [INF] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-07-15T06:55:42.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:41 smithi134 bash[24068]: audit 2024-07-15T06:55:41.639205+0000 mon.a (mon.0) 1044 : audit [INF] from='mgr.24544 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-07-15T06:55:42.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:41 smithi134 bash[24068]: audit 2024-07-15T06:55:41.642215+0000 mon.b (mon.1) 73 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-07-15T06:55:42.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:41 smithi134 bash[24068]: audit 2024-07-15T06:55:41.644026+0000 mon.b (mon.1) 74 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:55:42.425 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:55:42 smithi134 bash[59462]: logger=infra.usagestats.collector t=2024-07-15T06:55:42.11712969Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2024-07-15T06:55:42.426 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:55:42 smithi134 bash[59462]: logger=provisioning.datasources t=2024-07-15T06:55:42.23175492Z level=info msg="deleted correlations based on configuration" ds_name=Dashboard1 2024-07-15T06:55:42.426 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:55:42 smithi134 bash[59462]: logger=provisioning.datasources t=2024-07-15T06:55:42.231834262Z level=info msg="deleted datasource based on configuration" name=Dashboard1 2024-07-15T06:55:42.426 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:55:42 smithi134 bash[59462]: logger=provisioning.datasources t=2024-07-15T06:55:42.24326112Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2024-07-15T06:55:42.426 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:55:42 smithi134 bash[59462]: logger=provisioning.alerting t=2024-07-15T06:55:42.431441515Z level=info msg="starting to provision alerting" 2024-07-15T06:55:42.426 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:55:42 smithi134 bash[59462]: logger=provisioning.alerting t=2024-07-15T06:55:42.431499157Z level=info msg="finished to provision alerting" 2024-07-15T06:55:42.722 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:55:42 smithi134 bash[59462]: logger=grafanaStorageLogger t=2024-07-15T06:55:42.432217396Z level=info msg="storage starting" 2024-07-15T06:55:42.722 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:55:42 smithi134 bash[59462]: logger=ngalert.state.manager t=2024-07-15T06:55:42.43235839Z level=info msg="Warming state cache for startup" 2024-07-15T06:55:42.722 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:55:42 smithi134 bash[59462]: logger=http.server t=2024-07-15T06:55:42.434999857Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2024-07-15T06:55:42.722 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:55:42 smithi134 bash[59462]: logger=ngalert.state.manager t=2024-07-15T06:55:42.469736742Z level=info msg="State cache has been initialized" states=0 duration=37.378242ms 2024-07-15T06:55:42.722 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:55:42 smithi134 bash[59462]: logger=ngalert.multiorg.alertmanager t=2024-07-15T06:55:42.469807742Z level=info msg="starting MultiOrg Alertmanager" 2024-07-15T06:55:42.722 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:55:42 smithi134 bash[59462]: logger=ticker t=2024-07-15T06:55:42.469833635Z level=info msg=starting first_tick=2024-07-15T06:55:50Z 2024-07-15T06:55:43.053 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:42 smithi134 bash[24068]: cephadm 2024-07-15T06:55:41.624609+0000 mgr.x (mgr.24544) 54 : cephadm [INF] Upgrade: Updating mgr.y 2024-07-15T06:55:43.053 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:42 smithi134 bash[24068]: cephadm 2024-07-15T06:55:41.645904+0000 mgr.x (mgr.24544) 55 : cephadm [INF] Deploying daemon mgr.y on smithi089 2024-07-15T06:55:43.085 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:42 smithi089 bash[31372]: cephadm 2024-07-15T06:55:41.624609+0000 mgr.x (mgr.24544) 54 : cephadm [INF] Upgrade: Updating mgr.y 2024-07-15T06:55:43.085 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:42 smithi089 bash[31372]: cephadm 2024-07-15T06:55:41.645904+0000 mgr.x (mgr.24544) 55 : cephadm [INF] Deploying daemon mgr.y on smithi089 2024-07-15T06:55:43.085 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:42 smithi089 bash[21793]: cephadm 2024-07-15T06:55:41.624609+0000 mgr.x (mgr.24544) 54 : cephadm [INF] Upgrade: Updating mgr.y 2024-07-15T06:55:43.085 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:42 smithi089 bash[21793]: cephadm 2024-07-15T06:55:41.645904+0000 mgr.x (mgr.24544) 55 : cephadm [INF] Deploying daemon mgr.y on smithi089 2024-07-15T06:55:43.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:43 smithi134 bash[24068]: cluster 2024-07-15T06:55:42.531711+0000 mgr.x (mgr.24544) 56 : cluster [DBG] pgmap v24: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:55:43.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:43 smithi134 bash[24068]: audit 2024-07-15T06:55:43.445748+0000 mon.c (mon.2) 52 : audit [DBG] from='client.? 172.21.15.89:0/2691932586' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:55:43.804 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:55:43 smithi134 bash[58954]: ts=2024-07-15T06:55:43.507Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi134\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi134\", job=\"node\", machine=\"x86_64\", nodename=\"smithi134\", release=\"5.15.0-113-generic\", sysname=\"Linux\", version=\"#123-Ubuntu SMP Mon Jun 10 08:16:17 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", domainname=\"(none)\", instance=\"smithi134\", job=\"node\", machine=\"x86_64\", nodename=\"smithi134\", release=\"5.15.0-113-generic\", sysname=\"Linux\", version=\"#123-Ubuntu SMP Mon Jun 10 08:16:17 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:55:44.047 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:43 smithi089 bash[21793]: cluster 2024-07-15T06:55:42.531711+0000 mgr.x (mgr.24544) 56 : cluster [DBG] pgmap v24: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:55:44.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:43 smithi089 bash[21793]: audit 2024-07-15T06:55:43.445748+0000 mon.c (mon.2) 52 : audit [DBG] from='client.? 172.21.15.89:0/2691932586' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:55:44.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:43 smithi089 bash[31372]: cluster 2024-07-15T06:55:42.531711+0000 mgr.x (mgr.24544) 56 : cluster [DBG] pgmap v24: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:55:44.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:43 smithi089 bash[31372]: audit 2024-07-15T06:55:43.445748+0000 mon.c (mon.2) 52 : audit [DBG] from='client.? 172.21.15.89:0/2691932586' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:55:44.327 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:44 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:55:44.327 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:55:44 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:55:44.327 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:55:44 smithi089 systemd[1]: Stopping Ceph mgr.y for 75ba02bc-4274-11ef-bcad-c7b262605968... 2024-07-15T06:55:44.328 INFO:journalctl@ceph.osd.1.smithi089.stdout:Jul 15 06:55:44 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:55:44.328 INFO:journalctl@ceph.osd.3.smithi089.stdout:Jul 15 06:55:44 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:55:44.328 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:55:44 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:55:44.328 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:44 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:55:44.329 INFO:journalctl@ceph.osd.0.smithi089.stdout:Jul 15 06:55:44 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:55:44.329 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 06:55:44 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:55:44.330 INFO:journalctl@ceph.osd.2.smithi089.stdout:Jul 15 06:55:44 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:55:44.803 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:55:44 smithi134 bash[58954]: ts=2024-07-15T06:55:44.454Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:55:44.804 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:55:44 smithi134 bash[58954]: ts=2024-07-15T06:55:44.455Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:55:45.260 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:55:44 smithi089 bash[72779]: ceph-75ba02bc-4274-11ef-bcad-c7b262605968-mgr-y 2024-07-15T06:55:45.260 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:55:45 smithi089 systemd[1]: ceph-75ba02bc-4274-11ef-bcad-c7b262605968@mgr.y.service: Main process exited, code=exited, status=143/n/a 2024-07-15T06:55:45.260 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:55:45 smithi089 bash[72838]: Error response from daemon: No such container: ceph-75ba02bc-4274-11ef-bcad-c7b262605968-mgr-y 2024-07-15T06:55:45.260 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:55:45 smithi089 systemd[1]: ceph-75ba02bc-4274-11ef-bcad-c7b262605968@mgr.y.service: Failed with result 'exit-code'. 2024-07-15T06:55:45.261 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:55:45 smithi089 systemd[1]: Stopped Ceph mgr.y for 75ba02bc-4274-11ef-bcad-c7b262605968. 2024-07-15T06:55:45.619 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:45 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:55:45.619 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:55:45 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:55:45.619 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:55:45 smithi089 systemd[1]: Started Ceph mgr.y for 75ba02bc-4274-11ef-bcad-c7b262605968. 2024-07-15T06:55:45.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:45 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:55:45.620 INFO:journalctl@ceph.osd.0.smithi089.stdout:Jul 15 06:55:45 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:55:45.620 INFO:journalctl@ceph.osd.1.smithi089.stdout:Jul 15 06:55:45 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:55:45.620 INFO:journalctl@ceph.osd.3.smithi089.stdout:Jul 15 06:55:45 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:55:45.621 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:55:45 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:55:45.621 INFO:journalctl@ceph.osd.2.smithi089.stdout:Jul 15 06:55:45 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:55:45.621 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 06:55:45 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:55:46.053 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:45 smithi134 bash[24068]: cluster 2024-07-15T06:55:44.532336+0000 mgr.x (mgr.24544) 57 : cluster [DBG] pgmap v25: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:55:46.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:45 smithi134 bash[24068]: audit 2024-07-15T06:55:45.426071+0000 mon.a (mon.0) 1045 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:46.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:45 smithi134 bash[24068]: audit 2024-07-15T06:55:45.436784+0000 mon.a (mon.0) 1046 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:46.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:45 smithi134 bash[24068]: audit 2024-07-15T06:55:45.441865+0000 mon.b (mon.1) 75 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:55:46.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:45 smithi089 bash[21793]: cluster 2024-07-15T06:55:44.532336+0000 mgr.x (mgr.24544) 57 : cluster [DBG] pgmap v25: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:55:46.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:45 smithi089 bash[21793]: audit 2024-07-15T06:55:45.426071+0000 mon.a (mon.0) 1045 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:46.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:45 smithi089 bash[21793]: audit 2024-07-15T06:55:45.436784+0000 mon.a (mon.0) 1046 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:46.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:45 smithi089 bash[21793]: audit 2024-07-15T06:55:45.441865+0000 mon.b (mon.1) 75 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:55:46.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:45 smithi089 bash[31372]: cluster 2024-07-15T06:55:44.532336+0000 mgr.x (mgr.24544) 57 : cluster [DBG] pgmap v25: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:55:46.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:45 smithi089 bash[31372]: audit 2024-07-15T06:55:45.426071+0000 mon.a (mon.0) 1045 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:46.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:45 smithi089 bash[31372]: audit 2024-07-15T06:55:45.436784+0000 mon.a (mon.0) 1046 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:46.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:45 smithi089 bash[31372]: audit 2024-07-15T06:55:45.441865+0000 mon.b (mon.1) 75 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:55:47.053 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:46 smithi134 bash[24068]: audit 2024-07-15T06:55:45.805566+0000 mon.c (mon.2) 53 : audit [DBG] from='client.? 172.21.15.89:0/1763808358' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:55:47.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:46 smithi134 bash[24068]: audit 2024-07-15T06:55:46.080935+0000 mon.b (mon.1) 76 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:55:47.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:46 smithi089 bash[21793]: audit 2024-07-15T06:55:45.805566+0000 mon.c (mon.2) 53 : audit [DBG] from='client.? 172.21.15.89:0/1763808358' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:55:47.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:46 smithi089 bash[21793]: audit 2024-07-15T06:55:46.080935+0000 mon.b (mon.1) 76 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:55:47.119 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:55:46 smithi089 bash[72894]: debug 2024-07-15T06:55:46.801+0000 7f364faa5640 1 -- 172.21.15.89:0/1547855748 <== mon.2 v2:172.21.15.89:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55fdc4e6ad00 con 0x55fdc4e5d400 2024-07-15T06:55:47.120 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:55:46 smithi089 bash[72894]: debug 2024-07-15T06:55:46.917+0000 7f365230e040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-07-15T06:55:47.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:46 smithi089 bash[31372]: audit 2024-07-15T06:55:45.805566+0000 mon.c (mon.2) 53 : audit [DBG] from='client.? 172.21.15.89:0/1763808358' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:55:47.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:46 smithi089 bash[31372]: audit 2024-07-15T06:55:46.080935+0000 mon.b (mon.1) 76 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:55:48.053 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:47 smithi134 bash[24068]: cluster 2024-07-15T06:55:46.533004+0000 mgr.x (mgr.24544) 58 : cluster [DBG] pgmap v26: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:55:48.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:47 smithi089 bash[21793]: cluster 2024-07-15T06:55:46.533004+0000 mgr.x (mgr.24544) 58 : cluster [DBG] pgmap v26: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:55:48.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:47 smithi089 bash[31372]: cluster 2024-07-15T06:55:46.533004+0000 mgr.x (mgr.24544) 58 : cluster [DBG] pgmap v26: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:55:48.619 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:55:48 smithi089 bash[72894]: debug 2024-07-15T06:55:48.137+0000 7f365230e040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-07-15T06:55:48.619 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:55:48 smithi089 bash[72894]: debug 2024-07-15T06:55:48.205+0000 7f365230e040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-07-15T06:55:48.619 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:55:48 smithi089 bash[72894]: debug 2024-07-15T06:55:48.357+0000 7f365230e040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-07-15T06:55:49.013 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:48 smithi089 bash[21793]: audit 2024-07-15T06:55:48.158984+0000 mon.a (mon.0) 1047 : audit [DBG] from='client.? 172.21.15.89:0/1824228991' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:55:49.014 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:48 smithi089 bash[31372]: audit 2024-07-15T06:55:48.158984+0000 mon.a (mon.0) 1047 : audit [DBG] from='client.? 172.21.15.89:0/1824228991' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:55:49.053 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:48 smithi134 bash[24068]: audit 2024-07-15T06:55:48.158984+0000 mon.a (mon.0) 1047 : audit [DBG] from='client.? 172.21.15.89:0/1824228991' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:55:49.053 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:55:48 smithi134 bash[58954]: ts=2024-07-15T06:55:48.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1102-g9a202024 (9a2020246f85463d4fdb115c94aa40b463caa1d2) quincy (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1102-g9a202024 (9a2020246f85463d4fdb115c94aa40b463caa1d2) quincy (stable)\", cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:55:49.351 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:55:49 smithi089 bash[72894]: debug 2024-07-15T06:55:49.013+0000 7f365230e040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-07-15T06:55:49.351 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:55:49 smithi089 bash[72894]: debug 2024-07-15T06:55:49.085+0000 7f365230e040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-07-15T06:55:49.351 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:55:49 smithi089 bash[72894]: debug 2024-07-15T06:55:49.157+0000 7f365230e040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-07-15T06:55:49.619 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:55:49 smithi089 bash[72894]: debug 2024-07-15T06:55:49.353+0000 7f365230e040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-07-15T06:55:49.619 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:55:49 smithi089 bash[72894]: debug 2024-07-15T06:55:49.433+0000 7f365230e040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-07-15T06:55:49.619 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:55:49 smithi089 bash[72894]: debug 2024-07-15T06:55:49.501+0000 7f365230e040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-07-15T06:55:50.053 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:49 smithi134 bash[24068]: cluster 2024-07-15T06:55:48.534149+0000 mgr.x (mgr.24544) 59 : cluster [DBG] pgmap v27: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:55:50.054 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:55:49 smithi134 bash[52600]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:55:49] "GET /metrics HTTP/1.1" 200 33798 "" "Prometheus/2.43.0" 2024-07-15T06:55:50.075 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:49 smithi089 bash[21793]: cluster 2024-07-15T06:55:48.534149+0000 mgr.x (mgr.24544) 59 : cluster [DBG] pgmap v27: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:55:50.075 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:49 smithi089 bash[31372]: cluster 2024-07-15T06:55:48.534149+0000 mgr.x (mgr.24544) 59 : cluster [DBG] pgmap v27: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:55:50.075 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:55:49 smithi089 bash[72894]: debug 2024-07-15T06:55:49.761+0000 7f365230e040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-07-15T06:55:50.075 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:55:49 smithi089 bash[72894]: debug 2024-07-15T06:55:49.837+0000 7f365230e040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-07-15T06:55:50.075 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:55:50 smithi089 bash[72894]: debug 2024-07-15T06:55:50.077+0000 7f365230e040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-07-15T06:55:50.369 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:55:50 smithi089 bash[72894]: debug 2024-07-15T06:55:50.153+0000 7f365230e040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-07-15T06:55:50.754 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:55:50 smithi089 bash[72894]: debug 2024-07-15T06:55:50.417+0000 7f365230e040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-07-15T06:55:50.754 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:55:50 smithi089 bash[72894]: debug 2024-07-15T06:55:50.601+0000 7f365230e040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-07-15T06:55:51.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:50 smithi089 bash[21793]: audit 2024-07-15T06:55:50.514242+0000 mon.a (mon.0) 1048 : audit [DBG] from='client.? 172.21.15.89:0/1315281016' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:55:51.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:50 smithi089 bash[31372]: audit 2024-07-15T06:55:50.514242+0000 mon.a (mon.0) 1048 : audit [DBG] from='client.? 172.21.15.89:0/1315281016' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:55:51.031 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:55:50 smithi089 bash[72894]: debug 2024-07-15T06:55:50.769+0000 7f365230e040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-07-15T06:55:51.031 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:55:50 smithi089 bash[72894]: debug 2024-07-15T06:55:50.853+0000 7f365230e040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-07-15T06:55:51.053 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:50 smithi134 bash[24068]: audit 2024-07-15T06:55:50.514242+0000 mon.a (mon.0) 1048 : audit [DBG] from='client.? 172.21.15.89:0/1315281016' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:55:51.369 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:55:51 smithi089 bash[72894]: debug 2024-07-15T06:55:51.033+0000 7f365230e040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-07-15T06:55:52.009 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:55:51 smithi089 bash[72894]: debug 2024-07-15T06:55:51.925+0000 7f365230e040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-07-15T06:55:52.009 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:51 smithi089 bash[31372]: cluster 2024-07-15T06:55:50.534600+0000 mgr.x (mgr.24544) 60 : cluster [DBG] pgmap v28: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:55:52.009 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:51 smithi089 bash[31372]: audit 2024-07-15T06:55:51.252469+0000 mon.a (mon.0) 1049 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:52.009 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:51 smithi089 bash[31372]: audit 2024-07-15T06:55:51.262509+0000 mon.a (mon.0) 1050 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:52.009 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:51 smithi089 bash[21793]: cluster 2024-07-15T06:55:50.534600+0000 mgr.x (mgr.24544) 60 : cluster [DBG] pgmap v28: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:55:52.010 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:51 smithi089 bash[21793]: audit 2024-07-15T06:55:51.252469+0000 mon.a (mon.0) 1049 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:52.010 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:51 smithi089 bash[21793]: audit 2024-07-15T06:55:51.262509+0000 mon.a (mon.0) 1050 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:52.053 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:51 smithi134 bash[24068]: cluster 2024-07-15T06:55:50.534600+0000 mgr.x (mgr.24544) 60 : cluster [DBG] pgmap v28: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:55:52.053 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:51 smithi134 bash[24068]: audit 2024-07-15T06:55:51.252469+0000 mon.a (mon.0) 1049 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:52.053 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:51 smithi134 bash[24068]: audit 2024-07-15T06:55:51.262509+0000 mon.a (mon.0) 1050 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:52.324 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:55:52 smithi089 bash[72894]: debug 2024-07-15T06:55:52.009+0000 7f365230e040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-07-15T06:55:52.324 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:55:52 smithi089 bash[72894]: debug 2024-07-15T06:55:52.245+0000 7f365230e040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-07-15T06:55:52.324 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:55:52 smithi089 bash[72894]: debug 2024-07-15T06:55:52.325+0000 7f365230e040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-07-15T06:55:52.803 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:55:52 smithi134 bash[58954]: ts=2024-07-15T06:55:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:55:53.025 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:55:52 smithi089 bash[72894]: debug 2024-07-15T06:55:52.737+0000 7f365230e040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-07-15T06:55:53.025 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:55:52 smithi089 bash[72894]: debug 2024-07-15T06:55:52.825+0000 7f365230e040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-07-15T06:55:53.025 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:55:52 smithi089 bash[72894]: [15/Jul/2024:06:55:52] ENGINE Bus STARTING 2024-07-15T06:55:53.025 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:55:52 smithi089 bash[72894]: CherryPy Checker: 2024-07-15T06:55:53.025 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:55:52 smithi089 bash[72894]: The Application mounted at '' has an empty config. 2024-07-15T06:55:53.026 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:55:52 smithi089 bash[72894]: [15/Jul/2024:06:55:52] ENGINE Serving on http://:::9283 2024-07-15T06:55:53.026 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:55:52 smithi089 bash[72894]: [15/Jul/2024:06:55:52] ENGINE Bus STARTED 2024-07-15T06:55:53.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:53 smithi134 bash[24068]: audit 2024-07-15T06:55:52.030392+0000 mon.a (mon.0) 1051 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:53.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:53 smithi134 bash[24068]: audit 2024-07-15T06:55:52.042375+0000 mon.a (mon.0) 1052 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:53.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:53 smithi134 bash[24068]: cluster 2024-07-15T06:55:52.535636+0000 mgr.x (mgr.24544) 61 : cluster [DBG] pgmap v29: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:55:53.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:53 smithi134 bash[24068]: audit 2024-07-15T06:55:52.772374+0000 mon.a (mon.0) 1053 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:53.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:53 smithi134 bash[24068]: audit 2024-07-15T06:55:52.783687+0000 mon.a (mon.0) 1054 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:53.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:53 smithi134 bash[24068]: cluster 2024-07-15T06:55:52.849550+0000 mon.a (mon.0) 1055 : cluster [DBG] Standby manager daemon y restarted 2024-07-15T06:55:53.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:53 smithi134 bash[24068]: cluster 2024-07-15T06:55:52.849679+0000 mon.a (mon.0) 1056 : cluster [DBG] Standby manager daemon y started 2024-07-15T06:55:53.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:53 smithi134 bash[24068]: audit 2024-07-15T06:55:52.851215+0000 mon.c (mon.2) 54 : audit [DBG] from='mgr.? 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-07-15T06:55:53.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:53 smithi134 bash[24068]: audit 2024-07-15T06:55:52.852382+0000 mon.c (mon.2) 55 : audit [DBG] from='mgr.? 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-07-15T06:55:53.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:53 smithi134 bash[24068]: audit 2024-07-15T06:55:52.854278+0000 mon.c (mon.2) 56 : audit [DBG] from='mgr.? 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-07-15T06:55:53.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:53 smithi134 bash[24068]: audit 2024-07-15T06:55:52.855241+0000 mon.c (mon.2) 57 : audit [DBG] from='mgr.? 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-07-15T06:55:53.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:53 smithi134 bash[24068]: audit 2024-07-15T06:55:52.902278+0000 mon.a (mon.0) 1057 : audit [DBG] from='client.? 172.21.15.89:0/990390486' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:55:53.369 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:53 smithi089 bash[31372]: audit 2024-07-15T06:55:52.030392+0000 mon.a (mon.0) 1051 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:53.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:53 smithi089 bash[31372]: audit 2024-07-15T06:55:52.042375+0000 mon.a (mon.0) 1052 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:53.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:53 smithi089 bash[31372]: cluster 2024-07-15T06:55:52.535636+0000 mgr.x (mgr.24544) 61 : cluster [DBG] pgmap v29: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:55:53.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:53 smithi089 bash[31372]: audit 2024-07-15T06:55:52.772374+0000 mon.a (mon.0) 1053 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:53.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:53 smithi089 bash[31372]: audit 2024-07-15T06:55:52.783687+0000 mon.a (mon.0) 1054 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:53.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:53 smithi089 bash[31372]: cluster 2024-07-15T06:55:52.849550+0000 mon.a (mon.0) 1055 : cluster [DBG] Standby manager daemon y restarted 2024-07-15T06:55:53.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:53 smithi089 bash[31372]: cluster 2024-07-15T06:55:52.849679+0000 mon.a (mon.0) 1056 : cluster [DBG] Standby manager daemon y started 2024-07-15T06:55:53.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:53 smithi089 bash[31372]: audit 2024-07-15T06:55:52.851215+0000 mon.c (mon.2) 54 : audit [DBG] from='mgr.? 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-07-15T06:55:53.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:53 smithi089 bash[31372]: audit 2024-07-15T06:55:52.852382+0000 mon.c (mon.2) 55 : audit [DBG] from='mgr.? 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-07-15T06:55:53.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:53 smithi089 bash[31372]: audit 2024-07-15T06:55:52.854278+0000 mon.c (mon.2) 56 : audit [DBG] from='mgr.? 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-07-15T06:55:53.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:53 smithi089 bash[31372]: audit 2024-07-15T06:55:52.855241+0000 mon.c (mon.2) 57 : audit [DBG] from='mgr.? 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-07-15T06:55:53.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:53 smithi089 bash[31372]: audit 2024-07-15T06:55:52.902278+0000 mon.a (mon.0) 1057 : audit [DBG] from='client.? 172.21.15.89:0/990390486' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:55:53.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:53 smithi089 bash[21793]: audit 2024-07-15T06:55:52.030392+0000 mon.a (mon.0) 1051 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:53.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:53 smithi089 bash[21793]: audit 2024-07-15T06:55:52.042375+0000 mon.a (mon.0) 1052 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:53.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:53 smithi089 bash[21793]: cluster 2024-07-15T06:55:52.535636+0000 mgr.x (mgr.24544) 61 : cluster [DBG] pgmap v29: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:55:53.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:53 smithi089 bash[21793]: audit 2024-07-15T06:55:52.772374+0000 mon.a (mon.0) 1053 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:53.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:53 smithi089 bash[21793]: audit 2024-07-15T06:55:52.783687+0000 mon.a (mon.0) 1054 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:55:53.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:53 smithi089 bash[21793]: cluster 2024-07-15T06:55:52.849550+0000 mon.a (mon.0) 1055 : cluster [DBG] Standby manager daemon y restarted 2024-07-15T06:55:53.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:53 smithi089 bash[21793]: cluster 2024-07-15T06:55:52.849679+0000 mon.a (mon.0) 1056 : cluster [DBG] Standby manager daemon y started 2024-07-15T06:55:53.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:53 smithi089 bash[21793]: audit 2024-07-15T06:55:52.851215+0000 mon.c (mon.2) 54 : audit [DBG] from='mgr.? 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-07-15T06:55:53.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:53 smithi089 bash[21793]: audit 2024-07-15T06:55:52.852382+0000 mon.c (mon.2) 55 : audit [DBG] from='mgr.? 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-07-15T06:55:53.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:53 smithi089 bash[21793]: audit 2024-07-15T06:55:52.854278+0000 mon.c (mon.2) 56 : audit [DBG] from='mgr.? 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-07-15T06:55:53.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:53 smithi089 bash[21793]: audit 2024-07-15T06:55:52.855241+0000 mon.c (mon.2) 57 : audit [DBG] from='mgr.? 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-07-15T06:55:53.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:53 smithi089 bash[21793]: audit 2024-07-15T06:55:52.902278+0000 mon.a (mon.0) 1057 : audit [DBG] from='client.? 172.21.15.89:0/990390486' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:55:53.803 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:55:53 smithi134 bash[58954]: ts=2024-07-15T06:55:53.509Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi134\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi134\", job=\"node\", machine=\"x86_64\", nodename=\"smithi134\", release=\"5.15.0-113-generic\", sysname=\"Linux\", version=\"#123-Ubuntu SMP Mon Jun 10 08:16:17 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", domainname=\"(none)\", instance=\"smithi134\", job=\"node\", machine=\"x86_64\", nodename=\"smithi134\", release=\"5.15.0-113-generic\", sysname=\"Linux\", version=\"#123-Ubuntu SMP Mon Jun 10 08:16:17 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:55:54.793 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:55:54 smithi134 bash[58954]: ts=2024-07-15T06:55:54.454Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:55:54.793 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:55:54 smithi134 bash[58954]: ts=2024-07-15T06:55:54.455Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:55:55.053 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:54 smithi134 bash[24068]: cluster 2024-07-15T06:55:53.797575+0000 mon.a (mon.0) 1058 : cluster [DBG] mgrmap e27: x(active, since 53s), standbys: y 2024-07-15T06:55:55.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:54 smithi089 bash[31372]: cluster 2024-07-15T06:55:53.797575+0000 mon.a (mon.0) 1058 : cluster [DBG] mgrmap e27: x(active, since 53s), standbys: y 2024-07-15T06:55:55.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:54 smithi089 bash[21793]: cluster 2024-07-15T06:55:53.797575+0000 mon.a (mon.0) 1058 : cluster [DBG] mgrmap e27: x(active, since 53s), standbys: y 2024-07-15T06:55:56.053 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:55 smithi134 bash[24068]: cluster 2024-07-15T06:55:54.536278+0000 mgr.x (mgr.24544) 62 : cluster [DBG] pgmap v30: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:55:56.053 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:55 smithi134 bash[24068]: audit 2024-07-15T06:55:55.254467+0000 mon.c (mon.2) 58 : audit [DBG] from='client.? 172.21.15.89:0/1792051026' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:55:56.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:55 smithi089 bash[31372]: cluster 2024-07-15T06:55:54.536278+0000 mgr.x (mgr.24544) 62 : cluster [DBG] pgmap v30: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:55:56.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:55 smithi089 bash[31372]: audit 2024-07-15T06:55:55.254467+0000 mon.c (mon.2) 58 : audit [DBG] from='client.? 172.21.15.89:0/1792051026' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:55:56.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:55 smithi089 bash[21793]: cluster 2024-07-15T06:55:54.536278+0000 mgr.x (mgr.24544) 62 : cluster [DBG] pgmap v30: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:55:56.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:55 smithi089 bash[21793]: audit 2024-07-15T06:55:55.254467+0000 mon.c (mon.2) 58 : audit [DBG] from='client.? 172.21.15.89:0/1792051026' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:55:58.053 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:57 smithi134 bash[24068]: cluster 2024-07-15T06:55:56.536921+0000 mgr.x (mgr.24544) 63 : cluster [DBG] pgmap v31: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:55:58.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:57 smithi134 bash[24068]: audit 2024-07-15T06:55:57.627975+0000 mon.a (mon.0) 1059 : audit [DBG] from='client.? 172.21.15.89:0/3003615461' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:55:58.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:57 smithi089 bash[21793]: cluster 2024-07-15T06:55:56.536921+0000 mgr.x (mgr.24544) 63 : cluster [DBG] pgmap v31: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:55:58.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:57 smithi089 bash[21793]: audit 2024-07-15T06:55:57.627975+0000 mon.a (mon.0) 1059 : audit [DBG] from='client.? 172.21.15.89:0/3003615461' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:55:58.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:57 smithi089 bash[31372]: cluster 2024-07-15T06:55:56.536921+0000 mgr.x (mgr.24544) 63 : cluster [DBG] pgmap v31: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:55:58.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:57 smithi089 bash[31372]: audit 2024-07-15T06:55:57.627975+0000 mon.a (mon.0) 1059 : audit [DBG] from='client.? 172.21.15.89:0/3003615461' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:55:59.053 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:55:58 smithi134 bash[58954]: ts=2024-07-15T06:55:58.666Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1102-g9a202024 (9a2020246f85463d4fdb115c94aa40b463caa1d2) quincy (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1102-g9a202024 (9a2020246f85463d4fdb115c94aa40b463caa1d2) quincy (stable)\", cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:56:00.053 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:55:59 smithi134 bash[24068]: cluster 2024-07-15T06:55:58.538042+0000 mgr.x (mgr.24544) 64 : cluster [DBG] pgmap v32: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:56:00.054 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:55:59 smithi134 bash[52600]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:55:59] "GET /metrics HTTP/1.1" 200 33798 "" "Prometheus/2.43.0" 2024-07-15T06:56:00.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:55:59 smithi089 bash[21793]: cluster 2024-07-15T06:55:58.538042+0000 mgr.x (mgr.24544) 64 : cluster [DBG] pgmap v32: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:56:00.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:55:59 smithi089 bash[31372]: cluster 2024-07-15T06:55:58.538042+0000 mgr.x (mgr.24544) 64 : cluster [DBG] pgmap v32: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:56:01.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:00 smithi089 bash[21793]: audit 2024-07-15T06:55:59.991661+0000 mon.a (mon.0) 1060 : audit [DBG] from='client.? 172.21.15.89:0/3116085537' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:56:01.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:00 smithi089 bash[31372]: audit 2024-07-15T06:55:59.991661+0000 mon.a (mon.0) 1060 : audit [DBG] from='client.? 172.21.15.89:0/3116085537' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:56:01.303 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:00 smithi134 bash[24068]: audit 2024-07-15T06:55:59.991661+0000 mon.a (mon.0) 1060 : audit [DBG] from='client.? 172.21.15.89:0/3116085537' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:56:02.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:01 smithi089 bash[21793]: cluster 2024-07-15T06:56:00.538677+0000 mgr.x (mgr.24544) 65 : cluster [DBG] pgmap v33: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:56:02.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:01 smithi089 bash[21793]: audit 2024-07-15T06:56:01.081262+0000 mon.b (mon.1) 77 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:56:02.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:01 smithi089 bash[31372]: cluster 2024-07-15T06:56:00.538677+0000 mgr.x (mgr.24544) 65 : cluster [DBG] pgmap v33: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:56:02.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:01 smithi089 bash[31372]: audit 2024-07-15T06:56:01.081262+0000 mon.b (mon.1) 77 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:56:02.303 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:01 smithi134 bash[24068]: cluster 2024-07-15T06:56:00.538677+0000 mgr.x (mgr.24544) 65 : cluster [DBG] pgmap v33: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:56:02.303 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:01 smithi134 bash[24068]: audit 2024-07-15T06:56:01.081262+0000 mon.b (mon.1) 77 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:56:02.803 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:56:02 smithi134 bash[58954]: ts=2024-07-15T06:56:02.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:56:03.109 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:56:02 smithi134 bash[52600]: debug 2024-07-15T06:56:02.938+0000 7faacd15e640 -1 mgr handle_mgr_map I was active but no longer am 2024-07-15T06:56:03.109 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:56:02 smithi134 bash[52600]: ignoring --setuser ceph since I am not root 2024-07-15T06:56:03.109 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:56:02 smithi134 bash[52600]: ignoring --setgroup ceph since I am not root 2024-07-15T06:56:03.109 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:56:03 smithi134 bash[52600]: debug 2024-07-15T06:56:02.998+0000 7f835ee75640 1 -- 172.21.15.134:0/3884166602 <== mon.1 v2:172.21.15.134:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55ba593ecd00 con 0x55ba593de800 2024-07-15T06:56:03.110 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:02 smithi134 bash[24068]: cluster 2024-07-15T06:56:01.828942+0000 mon.a (mon.0) 1061 : cluster [DBG] mgrmap e28: x(active, since 61s), standbys: y 2024-07-15T06:56:03.110 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:02 smithi134 bash[24068]: audit 2024-07-15T06:56:02.359233+0000 mon.a (mon.0) 1062 : audit [DBG] from='client.? 172.21.15.89:0/444619545' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:56:03.110 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:02 smithi134 bash[24068]: audit 2024-07-15T06:56:02.724725+0000 mon.a (mon.0) 1063 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:56:03.110 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:02 smithi134 bash[24068]: audit 2024-07-15T06:56:02.739622+0000 mon.a (mon.0) 1064 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:56:03.110 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:02 smithi134 bash[24068]: audit 2024-07-15T06:56:02.744117+0000 mon.b (mon.1) 78 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:56:03.110 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:02 smithi134 bash[24068]: audit 2024-07-15T06:56:02.746326+0000 mon.b (mon.1) 79 : audit [INF] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:56:03.110 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:02 smithi134 bash[24068]: audit 2024-07-15T06:56:02.758267+0000 mon.a (mon.0) 1065 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:56:03.119 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:56:02 smithi089 bash[72894]: [15/Jul/2024:06:56:02] ENGINE Bus STOPPING 2024-07-15T06:56:03.119 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:56:02 smithi089 bash[72894]: [15/Jul/2024:06:56:02] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-07-15T06:56:03.119 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:56:02 smithi089 bash[72894]: [15/Jul/2024:06:56:02] ENGINE Bus STOPPED 2024-07-15T06:56:03.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:02 smithi089 bash[31372]: cluster 2024-07-15T06:56:01.828942+0000 mon.a (mon.0) 1061 : cluster [DBG] mgrmap e28: x(active, since 61s), standbys: y 2024-07-15T06:56:03.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:02 smithi089 bash[31372]: audit 2024-07-15T06:56:02.359233+0000 mon.a (mon.0) 1062 : audit [DBG] from='client.? 172.21.15.89:0/444619545' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:56:03.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:02 smithi089 bash[31372]: audit 2024-07-15T06:56:02.724725+0000 mon.a (mon.0) 1063 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:56:03.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:02 smithi089 bash[31372]: audit 2024-07-15T06:56:02.739622+0000 mon.a (mon.0) 1064 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:56:03.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:02 smithi089 bash[31372]: audit 2024-07-15T06:56:02.744117+0000 mon.b (mon.1) 78 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:56:03.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:02 smithi089 bash[31372]: audit 2024-07-15T06:56:02.746326+0000 mon.b (mon.1) 79 : audit [INF] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:56:03.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:02 smithi089 bash[31372]: audit 2024-07-15T06:56:02.758267+0000 mon.a (mon.0) 1065 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:56:03.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:02 smithi089 bash[21793]: cluster 2024-07-15T06:56:01.828942+0000 mon.a (mon.0) 1061 : cluster [DBG] mgrmap e28: x(active, since 61s), standbys: y 2024-07-15T06:56:03.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:02 smithi089 bash[21793]: audit 2024-07-15T06:56:02.359233+0000 mon.a (mon.0) 1062 : audit [DBG] from='client.? 172.21.15.89:0/444619545' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:56:03.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:02 smithi089 bash[21793]: audit 2024-07-15T06:56:02.724725+0000 mon.a (mon.0) 1063 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:56:03.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:02 smithi089 bash[21793]: audit 2024-07-15T06:56:02.739622+0000 mon.a (mon.0) 1064 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:56:03.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:02 smithi089 bash[21793]: audit 2024-07-15T06:56:02.744117+0000 mon.b (mon.1) 78 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:56:03.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:02 smithi089 bash[21793]: audit 2024-07-15T06:56:02.746326+0000 mon.b (mon.1) 79 : audit [INF] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:56:03.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:02 smithi089 bash[21793]: audit 2024-07-15T06:56:02.758267+0000 mon.a (mon.0) 1065 : audit [INF] from='mgr.24544 ' entity='mgr.x' 2024-07-15T06:56:03.365 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:56:03 smithi134 bash[52600]: debug 2024-07-15T06:56:03.110+0000 7f83616d0040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-07-15T06:56:03.498 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:56:03 smithi089 bash[72894]: [15/Jul/2024:06:56:03] ENGINE Bus STARTING 2024-07-15T06:56:03.498 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:56:03 smithi089 bash[72894]: debug 2024-07-15T06:56:03.229+0000 7f35b9930640 -1 client.0 error registering admin socket command: (17) File exists 2024-07-15T06:56:03.498 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:56:03 smithi089 bash[72894]: debug 2024-07-15T06:56:03.229+0000 7f35b9930640 -1 client.0 error registering admin socket command: (17) File exists 2024-07-15T06:56:03.498 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:56:03 smithi089 bash[72894]: debug 2024-07-15T06:56:03.229+0000 7f35b9930640 -1 client.0 error registering admin socket command: (17) File exists 2024-07-15T06:56:03.499 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:56:03 smithi089 bash[72894]: debug 2024-07-15T06:56:03.229+0000 7f35b9930640 -1 client.0 error registering admin socket command: (17) File exists 2024-07-15T06:56:03.499 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:56:03 smithi089 bash[72894]: debug 2024-07-15T06:56:03.229+0000 7f35b9930640 -1 client.0 error registering admin socket command: (17) File exists 2024-07-15T06:56:03.499 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:56:03 smithi089 bash[72894]: debug 2024-07-15T06:56:03.233+0000 7f35bc936640 -1 client.0 error registering admin socket command: (17) File exists 2024-07-15T06:56:03.499 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:56:03 smithi089 bash[72894]: debug 2024-07-15T06:56:03.233+0000 7f35bc936640 -1 client.0 error registering admin socket command: (17) File exists 2024-07-15T06:56:03.499 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:56:03 smithi089 bash[72894]: debug 2024-07-15T06:56:03.233+0000 7f35bc936640 -1 client.0 error registering admin socket command: (17) File exists 2024-07-15T06:56:03.499 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:56:03 smithi089 bash[72894]: debug 2024-07-15T06:56:03.233+0000 7f35bc936640 -1 client.0 error registering admin socket command: (17) File exists 2024-07-15T06:56:03.499 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:56:03 smithi089 bash[72894]: debug 2024-07-15T06:56:03.233+0000 7f35bc936640 -1 client.0 error registering admin socket command: (17) File exists 2024-07-15T06:56:03.499 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:56:03 smithi089 bash[72894]: [15/Jul/2024:06:56:03] ENGINE Serving on http://:::9283 2024-07-15T06:56:03.499 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:56:03 smithi089 bash[72894]: [15/Jul/2024:06:56:03] ENGINE Bus STARTED 2024-07-15T06:56:03.803 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:56:03 smithi134 bash[58954]: ts=2024-07-15T06:56:03.509Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi134\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi134\", job=\"node\", machine=\"x86_64\", nodename=\"smithi134\", release=\"5.15.0-113-generic\", sysname=\"Linux\", version=\"#123-Ubuntu SMP Mon Jun 10 08:16:17 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", domainname=\"(none)\", instance=\"smithi134\", job=\"node\", machine=\"x86_64\", nodename=\"smithi134\", release=\"5.15.0-113-generic\", sysname=\"Linux\", version=\"#123-Ubuntu SMP Mon Jun 10 08:16:17 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:56:04.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:03 smithi089 bash[31372]: cluster 2024-07-15T06:56:02.539863+0000 mgr.x (mgr.24544) 66 : cluster [DBG] pgmap v34: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s wr, 0 op/s 2024-07-15T06:56:04.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:03 smithi089 bash[31372]: audit 2024-07-15T06:56:02.915310+0000 mon.b (mon.1) 80 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:56:04.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:03 smithi089 bash[31372]: audit 2024-07-15T06:56:02.924181+0000 mon.a (mon.0) 1066 : audit [INF] from='mgr.24544 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-07-15T06:56:04.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:03 smithi089 bash[31372]: audit 2024-07-15T06:56:02.924440+0000 mon.b (mon.1) 81 : audit [INF] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-07-15T06:56:04.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:03 smithi089 bash[31372]: cluster 2024-07-15T06:56:02.924990+0000 mon.a (mon.0) 1067 : cluster [INF] Activating manager daemon y 2024-07-15T06:56:04.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:03 smithi089 bash[31372]: cluster 2024-07-15T06:56:02.939338+0000 mon.a (mon.0) 1068 : cluster [DBG] osdmap e78: 8 total, 8 up, 8 in 2024-07-15T06:56:04.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:03 smithi089 bash[31372]: audit 2024-07-15T06:56:02.939964+0000 mon.a (mon.0) 1069 : audit [INF] from='mgr.24544 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-07-15T06:56:04.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:03 smithi089 bash[31372]: cluster 2024-07-15T06:56:02.940124+0000 mon.a (mon.0) 1070 : cluster [DBG] mgrmap e29: y(active, starting, since 0.0151282s) 2024-07-15T06:56:04.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:03 smithi089 bash[31372]: audit 2024-07-15T06:56:02.948574+0000 mon.b (mon.1) 82 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-07-15T06:56:04.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:03 smithi089 bash[31372]: audit 2024-07-15T06:56:02.948812+0000 mon.b (mon.1) 83 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-15T06:56:04.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:03 smithi089 bash[31372]: audit 2024-07-15T06:56:02.949026+0000 mon.b (mon.1) 84 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-15T06:56:04.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:03 smithi089 bash[31372]: audit 2024-07-15T06:56:02.949893+0000 mon.b (mon.1) 85 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi134.wugtzh"}]: dispatch 2024-07-15T06:56:04.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:03 smithi089 bash[31372]: audit 2024-07-15T06:56:02.950221+0000 mon.b (mon.1) 86 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi089.cczuoz"}]: dispatch 2024-07-15T06:56:04.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:03 smithi089 bash[31372]: audit 2024-07-15T06:56:02.950967+0000 mon.b (mon.1) 87 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-07-15T06:56:04.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:03 smithi089 bash[31372]: audit 2024-07-15T06:56:02.951487+0000 mon.b (mon.1) 88 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-07-15T06:56:04.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:03 smithi089 bash[31372]: audit 2024-07-15T06:56:02.951817+0000 mon.b (mon.1) 89 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-07-15T06:56:04.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:03 smithi089 bash[31372]: audit 2024-07-15T06:56:02.952131+0000 mon.b (mon.1) 90 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-07-15T06:56:04.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:03 smithi089 bash[31372]: audit 2024-07-15T06:56:02.952483+0000 mon.b (mon.1) 91 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-07-15T06:56:04.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:03 smithi089 bash[31372]: audit 2024-07-15T06:56:02.952815+0000 mon.b (mon.1) 92 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-07-15T06:56:04.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:03 smithi089 bash[21793]: cluster 2024-07-15T06:56:02.539863+0000 mgr.x (mgr.24544) 66 : cluster [DBG] pgmap v34: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s wr, 0 op/s 2024-07-15T06:56:04.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:03 smithi089 bash[21793]: audit 2024-07-15T06:56:02.915310+0000 mon.b (mon.1) 80 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:56:04.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:03 smithi089 bash[21793]: audit 2024-07-15T06:56:02.924181+0000 mon.a (mon.0) 1066 : audit [INF] from='mgr.24544 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-07-15T06:56:04.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:03 smithi089 bash[21793]: audit 2024-07-15T06:56:02.924440+0000 mon.b (mon.1) 81 : audit [INF] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-07-15T06:56:04.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:03 smithi089 bash[21793]: cluster 2024-07-15T06:56:02.924990+0000 mon.a (mon.0) 1067 : cluster [INF] Activating manager daemon y 2024-07-15T06:56:04.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:03 smithi089 bash[21793]: cluster 2024-07-15T06:56:02.939338+0000 mon.a (mon.0) 1068 : cluster [DBG] osdmap e78: 8 total, 8 up, 8 in 2024-07-15T06:56:04.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:03 smithi089 bash[21793]: audit 2024-07-15T06:56:02.939964+0000 mon.a (mon.0) 1069 : audit [INF] from='mgr.24544 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-07-15T06:56:04.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:03 smithi089 bash[21793]: cluster 2024-07-15T06:56:02.940124+0000 mon.a (mon.0) 1070 : cluster [DBG] mgrmap e29: y(active, starting, since 0.0151282s) 2024-07-15T06:56:04.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:03 smithi089 bash[21793]: audit 2024-07-15T06:56:02.948574+0000 mon.b (mon.1) 82 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-07-15T06:56:04.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:03 smithi089 bash[21793]: audit 2024-07-15T06:56:02.948812+0000 mon.b (mon.1) 83 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-15T06:56:04.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:03 smithi089 bash[21793]: audit 2024-07-15T06:56:02.949026+0000 mon.b (mon.1) 84 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-15T06:56:04.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:03 smithi089 bash[21793]: audit 2024-07-15T06:56:02.949893+0000 mon.b (mon.1) 85 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi134.wugtzh"}]: dispatch 2024-07-15T06:56:04.123 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:03 smithi089 bash[21793]: audit 2024-07-15T06:56:02.950221+0000 mon.b (mon.1) 86 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi089.cczuoz"}]: dispatch 2024-07-15T06:56:04.123 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:03 smithi089 bash[21793]: audit 2024-07-15T06:56:02.950967+0000 mon.b (mon.1) 87 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-07-15T06:56:04.123 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:03 smithi089 bash[21793]: audit 2024-07-15T06:56:02.951487+0000 mon.b (mon.1) 88 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-07-15T06:56:04.123 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:03 smithi089 bash[21793]: audit 2024-07-15T06:56:02.951817+0000 mon.b (mon.1) 89 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-07-15T06:56:04.123 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:03 smithi089 bash[21793]: audit 2024-07-15T06:56:02.952131+0000 mon.b (mon.1) 90 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-07-15T06:56:04.123 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:03 smithi089 bash[21793]: audit 2024-07-15T06:56:02.952483+0000 mon.b (mon.1) 91 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-07-15T06:56:04.123 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:03 smithi089 bash[21793]: audit 2024-07-15T06:56:02.952815+0000 mon.b (mon.1) 92 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-07-15T06:56:04.123 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:03 smithi089 bash[31372]: audit 2024-07-15T06:56:02.953172+0000 mon.b (mon.1) 93 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-07-15T06:56:04.124 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:03 smithi089 bash[31372]: audit 2024-07-15T06:56:02.953513+0000 mon.b (mon.1) 94 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-07-15T06:56:04.124 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:03 smithi089 bash[31372]: audit 2024-07-15T06:56:02.953861+0000 mon.b (mon.1) 95 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-07-15T06:56:04.124 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:03 smithi089 bash[31372]: audit 2024-07-15T06:56:02.954213+0000 mon.b (mon.1) 96 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-07-15T06:56:04.124 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:03 smithi089 bash[31372]: audit 2024-07-15T06:56:02.954578+0000 mon.b (mon.1) 97 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-07-15T06:56:04.124 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:03 smithi089 bash[31372]: audit 2024-07-15T06:56:02.955389+0000 mon.b (mon.1) 98 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-07-15T06:56:04.124 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:03 smithi089 bash[31372]: cluster 2024-07-15T06:56:03.100128+0000 mon.a (mon.0) 1071 : cluster [INF] Manager daemon y is now available 2024-07-15T06:56:04.124 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:03 smithi089 bash[31372]: audit 2024-07-15T06:56:03.143711+0000 mon.b (mon.1) 99 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:56:04.124 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:03 smithi089 bash[31372]: audit 2024-07-15T06:56:03.171519+0000 mon.b (mon.1) 100 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:56:04.124 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:03 smithi089 bash[31372]: audit 2024-07-15T06:56:03.172587+0000 mon.a (mon.0) 1072 : audit [INF] from='mgr.25016 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-07-15T06:56:04.124 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:03 smithi089 bash[31372]: audit 2024-07-15T06:56:03.173109+0000 mon.b (mon.1) 101 : audit [INF] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-07-15T06:56:04.124 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:03 smithi089 bash[31372]: audit 2024-07-15T06:56:03.209612+0000 mon.a (mon.0) 1073 : audit [INF] from='mgr.25016 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-07-15T06:56:04.124 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:03 smithi089 bash[31372]: audit 2024-07-15T06:56:03.209831+0000 mon.b (mon.1) 102 : audit [INF] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-07-15T06:56:04.125 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:03 smithi089 bash[21793]: audit 2024-07-15T06:56:02.953172+0000 mon.b (mon.1) 93 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-07-15T06:56:04.125 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:03 smithi089 bash[21793]: audit 2024-07-15T06:56:02.953513+0000 mon.b (mon.1) 94 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-07-15T06:56:04.125 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:03 smithi089 bash[21793]: audit 2024-07-15T06:56:02.953861+0000 mon.b (mon.1) 95 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-07-15T06:56:04.125 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:03 smithi089 bash[21793]: audit 2024-07-15T06:56:02.954213+0000 mon.b (mon.1) 96 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-07-15T06:56:04.125 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:03 smithi089 bash[21793]: audit 2024-07-15T06:56:02.954578+0000 mon.b (mon.1) 97 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-07-15T06:56:04.125 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:03 smithi089 bash[21793]: audit 2024-07-15T06:56:02.955389+0000 mon.b (mon.1) 98 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-07-15T06:56:04.125 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:03 smithi089 bash[21793]: cluster 2024-07-15T06:56:03.100128+0000 mon.a (mon.0) 1071 : cluster [INF] Manager daemon y is now available 2024-07-15T06:56:04.125 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:03 smithi089 bash[21793]: audit 2024-07-15T06:56:03.143711+0000 mon.b (mon.1) 99 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:56:04.125 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:03 smithi089 bash[21793]: audit 2024-07-15T06:56:03.171519+0000 mon.b (mon.1) 100 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:56:04.125 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:03 smithi089 bash[21793]: audit 2024-07-15T06:56:03.172587+0000 mon.a (mon.0) 1072 : audit [INF] from='mgr.25016 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-07-15T06:56:04.125 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:03 smithi089 bash[21793]: audit 2024-07-15T06:56:03.173109+0000 mon.b (mon.1) 101 : audit [INF] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-07-15T06:56:04.125 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:03 smithi089 bash[21793]: audit 2024-07-15T06:56:03.209612+0000 mon.a (mon.0) 1073 : audit [INF] from='mgr.25016 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-07-15T06:56:04.125 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:03 smithi089 bash[21793]: audit 2024-07-15T06:56:03.209831+0000 mon.b (mon.1) 102 : audit [INF] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-07-15T06:56:04.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:03 smithi134 bash[24068]: cluster 2024-07-15T06:56:02.539863+0000 mgr.x (mgr.24544) 66 : cluster [DBG] pgmap v34: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s wr, 0 op/s 2024-07-15T06:56:04.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:03 smithi134 bash[24068]: audit 2024-07-15T06:56:02.915310+0000 mon.b (mon.1) 80 : audit [DBG] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:56:04.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:03 smithi134 bash[24068]: audit 2024-07-15T06:56:02.924181+0000 mon.a (mon.0) 1066 : audit [INF] from='mgr.24544 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-07-15T06:56:04.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:03 smithi134 bash[24068]: audit 2024-07-15T06:56:02.924440+0000 mon.b (mon.1) 81 : audit [INF] from='mgr.24544 172.21.15.134:0/1648136671' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-07-15T06:56:04.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:03 smithi134 bash[24068]: cluster 2024-07-15T06:56:02.924990+0000 mon.a (mon.0) 1067 : cluster [INF] Activating manager daemon y 2024-07-15T06:56:04.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:03 smithi134 bash[24068]: cluster 2024-07-15T06:56:02.939338+0000 mon.a (mon.0) 1068 : cluster [DBG] osdmap e78: 8 total, 8 up, 8 in 2024-07-15T06:56:04.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:03 smithi134 bash[24068]: audit 2024-07-15T06:56:02.939964+0000 mon.a (mon.0) 1069 : audit [INF] from='mgr.24544 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-07-15T06:56:04.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:03 smithi134 bash[24068]: cluster 2024-07-15T06:56:02.940124+0000 mon.a (mon.0) 1070 : cluster [DBG] mgrmap e29: y(active, starting, since 0.0151282s) 2024-07-15T06:56:04.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:03 smithi134 bash[24068]: audit 2024-07-15T06:56:02.948574+0000 mon.b (mon.1) 82 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-07-15T06:56:04.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:03 smithi134 bash[24068]: audit 2024-07-15T06:56:02.948812+0000 mon.b (mon.1) 83 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-15T06:56:04.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:03 smithi134 bash[24068]: audit 2024-07-15T06:56:02.949026+0000 mon.b (mon.1) 84 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-15T06:56:04.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:03 smithi134 bash[24068]: audit 2024-07-15T06:56:02.949893+0000 mon.b (mon.1) 85 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi134.wugtzh"}]: dispatch 2024-07-15T06:56:04.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:03 smithi134 bash[24068]: audit 2024-07-15T06:56:02.950221+0000 mon.b (mon.1) 86 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi089.cczuoz"}]: dispatch 2024-07-15T06:56:04.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:03 smithi134 bash[24068]: audit 2024-07-15T06:56:02.950967+0000 mon.b (mon.1) 87 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-07-15T06:56:04.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:03 smithi134 bash[24068]: audit 2024-07-15T06:56:02.951487+0000 mon.b (mon.1) 88 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-07-15T06:56:04.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:03 smithi134 bash[24068]: audit 2024-07-15T06:56:02.951817+0000 mon.b (mon.1) 89 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-07-15T06:56:04.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:03 smithi134 bash[24068]: audit 2024-07-15T06:56:02.952131+0000 mon.b (mon.1) 90 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-07-15T06:56:04.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:03 smithi134 bash[24068]: audit 2024-07-15T06:56:02.952483+0000 mon.b (mon.1) 91 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-07-15T06:56:04.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:03 smithi134 bash[24068]: audit 2024-07-15T06:56:02.952815+0000 mon.b (mon.1) 92 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-07-15T06:56:04.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:03 smithi134 bash[24068]: audit 2024-07-15T06:56:02.953172+0000 mon.b (mon.1) 93 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-07-15T06:56:04.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:03 smithi134 bash[24068]: audit 2024-07-15T06:56:02.953513+0000 mon.b (mon.1) 94 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-07-15T06:56:04.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:03 smithi134 bash[24068]: audit 2024-07-15T06:56:02.953861+0000 mon.b (mon.1) 95 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-07-15T06:56:04.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:03 smithi134 bash[24068]: audit 2024-07-15T06:56:02.954213+0000 mon.b (mon.1) 96 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-07-15T06:56:04.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:03 smithi134 bash[24068]: audit 2024-07-15T06:56:02.954578+0000 mon.b (mon.1) 97 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-07-15T06:56:04.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:03 smithi134 bash[24068]: audit 2024-07-15T06:56:02.955389+0000 mon.b (mon.1) 98 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-07-15T06:56:04.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:03 smithi134 bash[24068]: cluster 2024-07-15T06:56:03.100128+0000 mon.a (mon.0) 1071 : cluster [INF] Manager daemon y is now available 2024-07-15T06:56:04.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:03 smithi134 bash[24068]: audit 2024-07-15T06:56:03.143711+0000 mon.b (mon.1) 99 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:56:04.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:03 smithi134 bash[24068]: audit 2024-07-15T06:56:03.171519+0000 mon.b (mon.1) 100 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:56:04.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:03 smithi134 bash[24068]: audit 2024-07-15T06:56:03.172587+0000 mon.a (mon.0) 1072 : audit [INF] from='mgr.25016 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-07-15T06:56:04.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:03 smithi134 bash[24068]: audit 2024-07-15T06:56:03.173109+0000 mon.b (mon.1) 101 : audit [INF] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-07-15T06:56:04.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:03 smithi134 bash[24068]: audit 2024-07-15T06:56:03.209612+0000 mon.a (mon.0) 1073 : audit [INF] from='mgr.25016 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-07-15T06:56:04.307 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:03 smithi134 bash[24068]: audit 2024-07-15T06:56:03.209831+0000 mon.b (mon.1) 102 : audit [INF] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-07-15T06:56:04.803 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:56:04 smithi134 bash[52600]: debug 2024-07-15T06:56:04.366+0000 7f83616d0040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-07-15T06:56:04.804 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:56:04 smithi134 bash[52600]: debug 2024-07-15T06:56:04.430+0000 7f83616d0040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-07-15T06:56:04.804 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:56:04 smithi134 bash[52600]: debug 2024-07-15T06:56:04.586+0000 7f83616d0040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-07-15T06:56:04.804 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:56:04 smithi134 bash[58954]: ts=2024-07-15T06:56:04.454Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:56:04.805 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:56:04 smithi134 bash[58954]: ts=2024-07-15T06:56:04.455Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:56:04.870 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:56:04 smithi089 bash[72894]: debug 2024-07-15T06:56:04.372+0000 7f35a6109640 -1 log_channel(cephadm) log [ERR] : [15/Jul/2024:06:56:04] ENGINE Error in HTTPServer.serve 2024-07-15T06:56:04.870 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:56:04 smithi089 bash[72894]: Traceback (most recent call last): 2024-07-15T06:56:04.870 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:56:04 smithi089 bash[72894]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-07-15T06:56:04.870 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:56:04 smithi089 bash[72894]: self._connections.run(self.expiration_interval) 2024-07-15T06:56:04.870 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:56:04 smithi089 bash[72894]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-07-15T06:56:04.870 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:56:04 smithi089 bash[72894]: self._run(expiration_interval) 2024-07-15T06:56:04.871 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:56:04 smithi089 bash[72894]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-07-15T06:56:04.871 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:56:04 smithi089 bash[72894]: new_conn = self._from_server_socket(self.server.socket) 2024-07-15T06:56:04.871 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:56:04 smithi089 bash[72894]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-07-15T06:56:04.871 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:56:04 smithi089 bash[72894]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-07-15T06:56:04.871 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:56:04 smithi089 bash[72894]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-07-15T06:56:04.871 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:56:04 smithi089 bash[72894]: s = self.context.wrap_socket( 2024-07-15T06:56:04.871 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:56:04 smithi089 bash[72894]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-07-15T06:56:04.871 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:56:04 smithi089 bash[72894]: return self.sslsocket_class._create( 2024-07-15T06:56:04.871 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:56:04 smithi089 bash[72894]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-07-15T06:56:04.871 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:56:04 smithi089 bash[72894]: self.do_handshake() 2024-07-15T06:56:04.871 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:56:04 smithi089 bash[72894]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-07-15T06:56:04.871 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:56:04 smithi089 bash[72894]: self._sslobj.do_handshake() 2024-07-15T06:56:04.871 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:56:04 smithi089 bash[72894]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-07-15T06:56:05.248 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:04 smithi134 bash[24068]: cluster 2024-07-15T06:56:03.941998+0000 mon.a (mon.0) 1074 : cluster [DBG] mgrmap e30: y(active, since 1.01699s) 2024-07-15T06:56:05.248 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:04 smithi134 bash[24068]: cephadm 2024-07-15T06:56:04.258989+0000 mgr.y (mgr.25016) 2 : cephadm [INF] [15/Jul/2024:06:56:04] ENGINE Bus STARTING 2024-07-15T06:56:05.248 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:04 smithi134 bash[24068]: cephadm 2024-07-15T06:56:04.375839+0000 mgr.y (mgr.25016) 3 : cephadm [INF] [15/Jul/2024:06:56:04] ENGINE Serving on https://172.21.15.89:7150 2024-07-15T06:56:05.248 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:04 smithi134 bash[24068]: cephadm 2024-07-15T06:56:04.378908+0000 mgr.y (mgr.25016) 4 : cephadm [ERR] [15/Jul/2024:06:56:04] ENGINE Error in HTTPServer.serve 2024-07-15T06:56:05.248 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:04 smithi134 bash[24068]: Traceback (most recent call last): 2024-07-15T06:56:05.248 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:04 smithi134 bash[24068]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-07-15T06:56:05.248 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:04 smithi134 bash[24068]: self._connections.run(self.expiration_interval) 2024-07-15T06:56:05.248 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:04 smithi134 bash[24068]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-07-15T06:56:05.248 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:04 smithi134 bash[24068]: self._run(expiration_interval) 2024-07-15T06:56:05.248 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:04 smithi134 bash[24068]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-07-15T06:56:05.249 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:04 smithi134 bash[24068]: new_conn = self._from_server_socket(self.server.socket) 2024-07-15T06:56:05.249 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:04 smithi134 bash[24068]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-07-15T06:56:05.249 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:04 smithi134 bash[24068]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-07-15T06:56:05.249 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:04 smithi134 bash[24068]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-07-15T06:56:05.249 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:04 smithi134 bash[24068]: s = self.context.wrap_socket( 2024-07-15T06:56:05.249 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:04 smithi134 bash[24068]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-07-15T06:56:05.249 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:04 smithi134 bash[24068]: return self.sslsocket_class._create( 2024-07-15T06:56:05.249 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:04 smithi134 bash[24068]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-07-15T06:56:05.249 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:04 smithi134 bash[24068]: self.do_handshake() 2024-07-15T06:56:05.249 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:04 smithi134 bash[24068]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-07-15T06:56:05.249 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:04 smithi134 bash[24068]: self._sslobj.do_handshake() 2024-07-15T06:56:05.249 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:04 smithi134 bash[24068]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-07-15T06:56:05.249 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:04 smithi134 bash[24068]: cephadm 2024-07-15T06:56:04.477020+0000 mgr.y (mgr.25016) 5 : cephadm [INF] [15/Jul/2024:06:56:04] ENGINE Serving on http://172.21.15.89:8765 2024-07-15T06:56:05.249 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:04 smithi134 bash[24068]: cephadm 2024-07-15T06:56:04.477113+0000 mgr.y (mgr.25016) 6 : cephadm [INF] [15/Jul/2024:06:56:04] ENGINE Bus STARTED 2024-07-15T06:56:05.249 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:04 smithi134 bash[24068]: audit 2024-07-15T06:56:04.700727+0000 mon.c (mon.2) 59 : audit [DBG] from='client.? 172.21.15.89:0/942110223' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:56:05.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:04 smithi089 bash[21793]: cluster 2024-07-15T06:56:03.941998+0000 mon.a (mon.0) 1074 : cluster [DBG] mgrmap e30: y(active, since 1.01699s) 2024-07-15T06:56:05.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:04 smithi089 bash[21793]: cephadm 2024-07-15T06:56:04.258989+0000 mgr.y (mgr.25016) 2 : cephadm [INF] [15/Jul/2024:06:56:04] ENGINE Bus STARTING 2024-07-15T06:56:05.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:04 smithi089 bash[21793]: cephadm 2024-07-15T06:56:04.375839+0000 mgr.y (mgr.25016) 3 : cephadm [INF] [15/Jul/2024:06:56:04] ENGINE Serving on https://172.21.15.89:7150 2024-07-15T06:56:05.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:04 smithi089 bash[21793]: cephadm 2024-07-15T06:56:04.378908+0000 mgr.y (mgr.25016) 4 : cephadm [ERR] [15/Jul/2024:06:56:04] ENGINE Error in HTTPServer.serve 2024-07-15T06:56:05.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:04 smithi089 bash[21793]: Traceback (most recent call last): 2024-07-15T06:56:05.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:04 smithi089 bash[21793]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-07-15T06:56:05.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:04 smithi089 bash[21793]: self._connections.run(self.expiration_interval) 2024-07-15T06:56:05.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:04 smithi089 bash[21793]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-07-15T06:56:05.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:04 smithi089 bash[21793]: self._run(expiration_interval) 2024-07-15T06:56:05.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:04 smithi089 bash[21793]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-07-15T06:56:05.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:04 smithi089 bash[21793]: new_conn = self._from_server_socket(self.server.socket) 2024-07-15T06:56:05.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:04 smithi089 bash[21793]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-07-15T06:56:05.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:04 smithi089 bash[21793]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-07-15T06:56:05.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:04 smithi089 bash[21793]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-07-15T06:56:05.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:04 smithi089 bash[21793]: s = self.context.wrap_socket( 2024-07-15T06:56:05.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:04 smithi089 bash[21793]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-07-15T06:56:05.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:04 smithi089 bash[21793]: return self.sslsocket_class._create( 2024-07-15T06:56:05.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:04 smithi089 bash[21793]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-07-15T06:56:05.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:04 smithi089 bash[21793]: self.do_handshake() 2024-07-15T06:56:05.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:04 smithi089 bash[21793]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-07-15T06:56:05.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:04 smithi089 bash[21793]: self._sslobj.do_handshake() 2024-07-15T06:56:05.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:04 smithi089 bash[21793]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-07-15T06:56:05.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:04 smithi089 bash[21793]: cephadm 2024-07-15T06:56:04.477020+0000 mgr.y (mgr.25016) 5 : cephadm [INF] [15/Jul/2024:06:56:04] ENGINE Serving on http://172.21.15.89:8765 2024-07-15T06:56:05.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:04 smithi089 bash[21793]: cephadm 2024-07-15T06:56:04.477113+0000 mgr.y (mgr.25016) 6 : cephadm [INF] [15/Jul/2024:06:56:04] ENGINE Bus STARTED 2024-07-15T06:56:05.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:04 smithi089 bash[21793]: audit 2024-07-15T06:56:04.700727+0000 mon.c (mon.2) 59 : audit [DBG] from='client.? 172.21.15.89:0/942110223' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:56:05.372 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:04 smithi089 bash[31372]: cluster 2024-07-15T06:56:03.941998+0000 mon.a (mon.0) 1074 : cluster [DBG] mgrmap e30: y(active, since 1.01699s) 2024-07-15T06:56:05.372 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:04 smithi089 bash[31372]: cephadm 2024-07-15T06:56:04.258989+0000 mgr.y (mgr.25016) 2 : cephadm [INF] [15/Jul/2024:06:56:04] ENGINE Bus STARTING 2024-07-15T06:56:05.372 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:04 smithi089 bash[31372]: cephadm 2024-07-15T06:56:04.375839+0000 mgr.y (mgr.25016) 3 : cephadm [INF] [15/Jul/2024:06:56:04] ENGINE Serving on https://172.21.15.89:7150 2024-07-15T06:56:05.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:04 smithi089 bash[31372]: cephadm 2024-07-15T06:56:04.378908+0000 mgr.y (mgr.25016) 4 : cephadm [ERR] [15/Jul/2024:06:56:04] ENGINE Error in HTTPServer.serve 2024-07-15T06:56:05.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:04 smithi089 bash[31372]: Traceback (most recent call last): 2024-07-15T06:56:05.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:04 smithi089 bash[31372]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-07-15T06:56:05.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:04 smithi089 bash[31372]: self._connections.run(self.expiration_interval) 2024-07-15T06:56:05.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:04 smithi089 bash[31372]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-07-15T06:56:05.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:04 smithi089 bash[31372]: self._run(expiration_interval) 2024-07-15T06:56:05.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:04 smithi089 bash[31372]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-07-15T06:56:05.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:04 smithi089 bash[31372]: new_conn = self._from_server_socket(self.server.socket) 2024-07-15T06:56:05.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:04 smithi089 bash[31372]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-07-15T06:56:05.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:04 smithi089 bash[31372]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-07-15T06:56:05.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:04 smithi089 bash[31372]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-07-15T06:56:05.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:04 smithi089 bash[31372]: s = self.context.wrap_socket( 2024-07-15T06:56:05.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:04 smithi089 bash[31372]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-07-15T06:56:05.374 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:04 smithi089 bash[31372]: return self.sslsocket_class._create( 2024-07-15T06:56:05.374 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:04 smithi089 bash[31372]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-07-15T06:56:05.374 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:04 smithi089 bash[31372]: self.do_handshake() 2024-07-15T06:56:05.374 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:04 smithi089 bash[31372]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-07-15T06:56:05.374 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:04 smithi089 bash[31372]: self._sslobj.do_handshake() 2024-07-15T06:56:05.374 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:04 smithi089 bash[31372]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-07-15T06:56:05.374 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:04 smithi089 bash[31372]: cephadm 2024-07-15T06:56:04.477020+0000 mgr.y (mgr.25016) 5 : cephadm [INF] [15/Jul/2024:06:56:04] ENGINE Serving on http://172.21.15.89:8765 2024-07-15T06:56:05.374 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:04 smithi089 bash[31372]: cephadm 2024-07-15T06:56:04.477113+0000 mgr.y (mgr.25016) 6 : cephadm [INF] [15/Jul/2024:06:56:04] ENGINE Bus STARTED 2024-07-15T06:56:05.374 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:04 smithi089 bash[31372]: audit 2024-07-15T06:56:04.700727+0000 mon.c (mon.2) 59 : audit [DBG] from='client.? 172.21.15.89:0/942110223' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:56:05.553 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:56:05 smithi134 bash[52600]: debug 2024-07-15T06:56:05.250+0000 7f83616d0040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-07-15T06:56:05.553 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:56:05 smithi134 bash[52600]: debug 2024-07-15T06:56:05.326+0000 7f83616d0040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-07-15T06:56:05.553 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:56:05 smithi134 bash[52600]: debug 2024-07-15T06:56:05.394+0000 7f83616d0040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-07-15T06:56:05.988 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:56:05 smithi134 bash[52600]: debug 2024-07-15T06:56:05.582+0000 7f83616d0040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-07-15T06:56:05.988 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:56:05 smithi134 bash[52600]: debug 2024-07-15T06:56:05.658+0000 7f83616d0040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-07-15T06:56:05.988 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:56:05 smithi134 bash[52600]: debug 2024-07-15T06:56:05.730+0000 7f83616d0040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-07-15T06:56:06.258 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:56:05 smithi134 bash[52600]: debug 2024-07-15T06:56:05.994+0000 7f83616d0040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-07-15T06:56:06.258 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:56:06 smithi134 bash[52600]: debug 2024-07-15T06:56:06.062+0000 7f83616d0040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-07-15T06:56:06.259 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:05 smithi134 bash[24068]: cluster 2024-07-15T06:56:04.952366+0000 mgr.y (mgr.25016) 7 : cluster [DBG] pgmap v4: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:56:06.259 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:05 smithi134 bash[24068]: cluster 2024-07-15T06:56:04.996827+0000 mon.a (mon.0) 1075 : cluster [DBG] mgrmap e31: y(active, since 2s) 2024-07-15T06:56:06.369 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:05 smithi089 bash[21793]: cluster 2024-07-15T06:56:04.952366+0000 mgr.y (mgr.25016) 7 : cluster [DBG] pgmap v4: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:56:06.369 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:05 smithi089 bash[21793]: cluster 2024-07-15T06:56:04.996827+0000 mon.a (mon.0) 1075 : cluster [DBG] mgrmap e31: y(active, since 2s) 2024-07-15T06:56:06.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:05 smithi089 bash[31372]: cluster 2024-07-15T06:56:04.952366+0000 mgr.y (mgr.25016) 7 : cluster [DBG] pgmap v4: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:56:06.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:05 smithi089 bash[31372]: cluster 2024-07-15T06:56:04.996827+0000 mon.a (mon.0) 1075 : cluster [DBG] mgrmap e31: y(active, since 2s) 2024-07-15T06:56:06.553 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:56:06 smithi134 bash[52600]: debug 2024-07-15T06:56:06.258+0000 7f83616d0040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-07-15T06:56:06.553 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:56:06 smithi134 bash[52600]: debug 2024-07-15T06:56:06.338+0000 7f83616d0040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-07-15T06:56:06.942 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:56:06 smithi134 bash[52600]: debug 2024-07-15T06:56:06.598+0000 7f83616d0040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-07-15T06:56:06.942 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:56:06 smithi134 bash[52600]: debug 2024-07-15T06:56:06.774+0000 7f83616d0040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-07-15T06:56:07.205 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:56:06 smithi134 bash[52600]: debug 2024-07-15T06:56:06.942+0000 7f83616d0040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-07-15T06:56:07.205 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:56:07 smithi134 bash[52600]: debug 2024-07-15T06:56:07.026+0000 7f83616d0040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-07-15T06:56:07.553 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:56:07 smithi134 bash[52600]: debug 2024-07-15T06:56:07.206+0000 7f83616d0040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-07-15T06:56:08.303 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:56:08 smithi134 bash[52600]: debug 2024-07-15T06:56:08.038+0000 7f83616d0040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-07-15T06:56:08.303 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:56:08 smithi134 bash[52600]: debug 2024-07-15T06:56:08.114+0000 7f83616d0040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-07-15T06:56:08.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:08 smithi134 bash[24068]: cluster 2024-07-15T06:56:06.952908+0000 mgr.y (mgr.25016) 8 : cluster [DBG] pgmap v5: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:56:08.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:08 smithi134 bash[24068]: cluster 2024-07-15T06:56:07.008035+0000 mon.a (mon.0) 1076 : cluster [DBG] mgrmap e32: y(active, since 4s) 2024-07-15T06:56:08.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:08 smithi134 bash[24068]: audit 2024-07-15T06:56:07.072036+0000 mon.a (mon.0) 1077 : audit [DBG] from='client.? 172.21.15.89:0/1838048595' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:56:08.369 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:08 smithi089 bash[21793]: cluster 2024-07-15T06:56:06.952908+0000 mgr.y (mgr.25016) 8 : cluster [DBG] pgmap v5: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:56:08.369 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:08 smithi089 bash[21793]: cluster 2024-07-15T06:56:07.008035+0000 mon.a (mon.0) 1076 : cluster [DBG] mgrmap e32: y(active, since 4s) 2024-07-15T06:56:08.369 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:08 smithi089 bash[21793]: audit 2024-07-15T06:56:07.072036+0000 mon.a (mon.0) 1077 : audit [DBG] from='client.? 172.21.15.89:0/1838048595' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:56:08.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:08 smithi089 bash[31372]: cluster 2024-07-15T06:56:06.952908+0000 mgr.y (mgr.25016) 8 : cluster [DBG] pgmap v5: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:56:08.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:08 smithi089 bash[31372]: cluster 2024-07-15T06:56:07.008035+0000 mon.a (mon.0) 1076 : cluster [DBG] mgrmap e32: y(active, since 4s) 2024-07-15T06:56:08.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:08 smithi089 bash[31372]: audit 2024-07-15T06:56:07.072036+0000 mon.a (mon.0) 1077 : audit [DBG] from='client.? 172.21.15.89:0/1838048595' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:56:08.653 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:56:08 smithi134 bash[52600]: debug 2024-07-15T06:56:08.338+0000 7f83616d0040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-07-15T06:56:08.653 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:56:08 smithi134 bash[52600]: debug 2024-07-15T06:56:08.414+0000 7f83616d0040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-07-15T06:56:08.918 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:56:08 smithi134 bash[52600]: debug 2024-07-15T06:56:08.830+0000 7f83616d0040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-07-15T06:56:08.918 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:56:08 smithi134 bash[58954]: ts=2024-07-15T06:56:08.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1102-g9a202024 (9a2020246f85463d4fdb115c94aa40b463caa1d2) quincy (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1102-g9a202024 (9a2020246f85463d4fdb115c94aa40b463caa1d2) quincy (stable)\", cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:56:09.303 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:09 smithi134 bash[24068]: cluster 2024-07-15T06:56:08.946207+0000 mon.a (mon.0) 1078 : cluster [DBG] Standby manager daemon x started 2024-07-15T06:56:09.303 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:09 smithi134 bash[24068]: audit 2024-07-15T06:56:08.948575+0000 mon.b (mon.1) 103 : audit [DBG] from='mgr.? 172.21.15.134:0/1066400495' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-07-15T06:56:09.303 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:09 smithi134 bash[24068]: audit 2024-07-15T06:56:08.949596+0000 mon.b (mon.1) 104 : audit [DBG] from='mgr.? 172.21.15.134:0/1066400495' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-07-15T06:56:09.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:09 smithi134 bash[24068]: audit 2024-07-15T06:56:08.951700+0000 mon.b (mon.1) 105 : audit [DBG] from='mgr.? 172.21.15.134:0/1066400495' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-07-15T06:56:09.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:09 smithi134 bash[24068]: audit 2024-07-15T06:56:08.952086+0000 mon.b (mon.1) 106 : audit [DBG] from='mgr.? 172.21.15.134:0/1066400495' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-07-15T06:56:09.304 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:56:08 smithi134 bash[52600]: debug 2024-07-15T06:56:08.918+0000 7f83616d0040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-07-15T06:56:09.304 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:56:08 smithi134 bash[52600]: [15/Jul/2024:06:56:08] ENGINE Bus STARTING 2024-07-15T06:56:09.304 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:56:08 smithi134 bash[52600]: CherryPy Checker: 2024-07-15T06:56:09.304 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:56:08 smithi134 bash[52600]: The Application mounted at '' has an empty config. 2024-07-15T06:56:09.304 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:56:09 smithi134 bash[52600]: [15/Jul/2024:06:56:09] ENGINE Serving on http://:::9283 2024-07-15T06:56:09.304 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:56:09 smithi134 bash[52600]: [15/Jul/2024:06:56:09] ENGINE Bus STARTED 2024-07-15T06:56:09.369 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:09 smithi089 bash[21793]: cluster 2024-07-15T06:56:08.946207+0000 mon.a (mon.0) 1078 : cluster [DBG] Standby manager daemon x started 2024-07-15T06:56:09.369 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:09 smithi089 bash[21793]: audit 2024-07-15T06:56:08.948575+0000 mon.b (mon.1) 103 : audit [DBG] from='mgr.? 172.21.15.134:0/1066400495' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-07-15T06:56:09.369 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:09 smithi089 bash[21793]: audit 2024-07-15T06:56:08.949596+0000 mon.b (mon.1) 104 : audit [DBG] from='mgr.? 172.21.15.134:0/1066400495' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-07-15T06:56:09.369 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:09 smithi089 bash[21793]: audit 2024-07-15T06:56:08.951700+0000 mon.b (mon.1) 105 : audit [DBG] from='mgr.? 172.21.15.134:0/1066400495' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-07-15T06:56:09.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:09 smithi089 bash[21793]: audit 2024-07-15T06:56:08.952086+0000 mon.b (mon.1) 106 : audit [DBG] from='mgr.? 172.21.15.134:0/1066400495' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-07-15T06:56:09.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:09 smithi089 bash[31372]: cluster 2024-07-15T06:56:08.946207+0000 mon.a (mon.0) 1078 : cluster [DBG] Standby manager daemon x started 2024-07-15T06:56:09.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:09 smithi089 bash[31372]: audit 2024-07-15T06:56:08.948575+0000 mon.b (mon.1) 103 : audit [DBG] from='mgr.? 172.21.15.134:0/1066400495' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-07-15T06:56:09.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:09 smithi089 bash[31372]: audit 2024-07-15T06:56:08.949596+0000 mon.b (mon.1) 104 : audit [DBG] from='mgr.? 172.21.15.134:0/1066400495' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-07-15T06:56:09.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:09 smithi089 bash[31372]: audit 2024-07-15T06:56:08.951700+0000 mon.b (mon.1) 105 : audit [DBG] from='mgr.? 172.21.15.134:0/1066400495' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-07-15T06:56:09.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:09 smithi089 bash[31372]: audit 2024-07-15T06:56:08.952086+0000 mon.b (mon.1) 106 : audit [DBG] from='mgr.? 172.21.15.134:0/1066400495' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-07-15T06:56:10.053 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:56:09 smithi134 bash[52600]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:56:09] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-07-15T06:56:10.053 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:10 smithi134 bash[24068]: cluster 2024-07-15T06:56:08.953288+0000 mgr.y (mgr.25016) 9 : cluster [DBG] pgmap v6: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:56:10.053 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:10 smithi134 bash[24068]: audit 2024-07-15T06:56:09.016819+0000 mon.b (mon.1) 107 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-07-15T06:56:10.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:10 smithi134 bash[24068]: cluster 2024-07-15T06:56:09.017454+0000 mon.a (mon.0) 1079 : cluster [DBG] mgrmap e33: y(active, since 6s), standbys: x 2024-07-15T06:56:10.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:10 smithi134 bash[24068]: audit 2024-07-15T06:56:09.076823+0000 mon.a (mon.0) 1080 : audit [INF] from='mgr.25016 ' entity='mgr.y' 2024-07-15T06:56:10.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:10 smithi134 bash[24068]: audit 2024-07-15T06:56:09.084504+0000 mon.a (mon.0) 1081 : audit [INF] from='mgr.25016 ' entity='mgr.y' 2024-07-15T06:56:10.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:10 smithi134 bash[24068]: audit 2024-07-15T06:56:09.377364+0000 mon.a (mon.0) 1082 : audit [INF] from='mgr.25016 ' entity='mgr.y' 2024-07-15T06:56:10.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:10 smithi134 bash[24068]: audit 2024-07-15T06:56:09.386464+0000 mon.a (mon.0) 1083 : audit [INF] from='mgr.25016 ' entity='mgr.y' 2024-07-15T06:56:10.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:10 smithi134 bash[24068]: audit 2024-07-15T06:56:09.414134+0000 mon.a (mon.0) 1084 : audit [DBG] from='client.? 172.21.15.89:0/4061045744' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:56:10.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:10 smithi134 bash[24068]: audit 2024-07-15T06:56:09.787277+0000 mon.a (mon.0) 1085 : audit [INF] from='mgr.25016 ' entity='mgr.y' 2024-07-15T06:56:10.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:10 smithi134 bash[24068]: audit 2024-07-15T06:56:09.795335+0000 mon.a (mon.0) 1086 : audit [INF] from='mgr.25016 ' entity='mgr.y' 2024-07-15T06:56:10.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:10 smithi089 bash[21793]: cluster 2024-07-15T06:56:08.953288+0000 mgr.y (mgr.25016) 9 : cluster [DBG] pgmap v6: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:56:10.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:10 smithi089 bash[21793]: audit 2024-07-15T06:56:09.016819+0000 mon.b (mon.1) 107 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-07-15T06:56:10.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:10 smithi089 bash[21793]: cluster 2024-07-15T06:56:09.017454+0000 mon.a (mon.0) 1079 : cluster [DBG] mgrmap e33: y(active, since 6s), standbys: x 2024-07-15T06:56:10.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:10 smithi089 bash[21793]: audit 2024-07-15T06:56:09.076823+0000 mon.a (mon.0) 1080 : audit [INF] from='mgr.25016 ' entity='mgr.y' 2024-07-15T06:56:10.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:10 smithi089 bash[21793]: audit 2024-07-15T06:56:09.084504+0000 mon.a (mon.0) 1081 : audit [INF] from='mgr.25016 ' entity='mgr.y' 2024-07-15T06:56:10.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:10 smithi089 bash[21793]: audit 2024-07-15T06:56:09.377364+0000 mon.a (mon.0) 1082 : audit [INF] from='mgr.25016 ' entity='mgr.y' 2024-07-15T06:56:10.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:10 smithi089 bash[21793]: audit 2024-07-15T06:56:09.386464+0000 mon.a (mon.0) 1083 : audit [INF] from='mgr.25016 ' entity='mgr.y' 2024-07-15T06:56:10.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:10 smithi089 bash[21793]: audit 2024-07-15T06:56:09.414134+0000 mon.a (mon.0) 1084 : audit [DBG] from='client.? 172.21.15.89:0/4061045744' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:56:10.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:10 smithi089 bash[21793]: audit 2024-07-15T06:56:09.787277+0000 mon.a (mon.0) 1085 : audit [INF] from='mgr.25016 ' entity='mgr.y' 2024-07-15T06:56:10.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:10 smithi089 bash[21793]: audit 2024-07-15T06:56:09.795335+0000 mon.a (mon.0) 1086 : audit [INF] from='mgr.25016 ' entity='mgr.y' 2024-07-15T06:56:10.371 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:10 smithi089 bash[31372]: cluster 2024-07-15T06:56:08.953288+0000 mgr.y (mgr.25016) 9 : cluster [DBG] pgmap v6: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:56:10.371 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:10 smithi089 bash[31372]: audit 2024-07-15T06:56:09.016819+0000 mon.b (mon.1) 107 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-07-15T06:56:10.371 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:10 smithi089 bash[31372]: cluster 2024-07-15T06:56:09.017454+0000 mon.a (mon.0) 1079 : cluster [DBG] mgrmap e33: y(active, since 6s), standbys: x 2024-07-15T06:56:10.371 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:10 smithi089 bash[31372]: audit 2024-07-15T06:56:09.076823+0000 mon.a (mon.0) 1080 : audit [INF] from='mgr.25016 ' entity='mgr.y' 2024-07-15T06:56:10.371 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:10 smithi089 bash[31372]: audit 2024-07-15T06:56:09.084504+0000 mon.a (mon.0) 1081 : audit [INF] from='mgr.25016 ' entity='mgr.y' 2024-07-15T06:56:10.371 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:10 smithi089 bash[31372]: audit 2024-07-15T06:56:09.377364+0000 mon.a (mon.0) 1082 : audit [INF] from='mgr.25016 ' entity='mgr.y' 2024-07-15T06:56:10.371 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:10 smithi089 bash[31372]: audit 2024-07-15T06:56:09.386464+0000 mon.a (mon.0) 1083 : audit [INF] from='mgr.25016 ' entity='mgr.y' 2024-07-15T06:56:10.371 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:10 smithi089 bash[31372]: audit 2024-07-15T06:56:09.414134+0000 mon.a (mon.0) 1084 : audit [DBG] from='client.? 172.21.15.89:0/4061045744' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:56:10.371 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:10 smithi089 bash[31372]: audit 2024-07-15T06:56:09.787277+0000 mon.a (mon.0) 1085 : audit [INF] from='mgr.25016 ' entity='mgr.y' 2024-07-15T06:56:10.371 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:10 smithi089 bash[31372]: audit 2024-07-15T06:56:09.795335+0000 mon.a (mon.0) 1086 : audit [INF] from='mgr.25016 ' entity='mgr.y' 2024-07-15T06:56:11.303 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:11 smithi134 bash[24068]: audit 2024-07-15T06:56:10.050965+0000 mon.a (mon.0) 1087 : audit [INF] from='mgr.25016 ' entity='mgr.y' 2024-07-15T06:56:11.303 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:11 smithi134 bash[24068]: audit 2024-07-15T06:56:10.058584+0000 mon.a (mon.0) 1088 : audit [INF] from='mgr.25016 ' entity='mgr.y' 2024-07-15T06:56:11.369 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:11 smithi089 bash[21793]: audit 2024-07-15T06:56:10.050965+0000 mon.a (mon.0) 1087 : audit [INF] from='mgr.25016 ' entity='mgr.y' 2024-07-15T06:56:11.369 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:11 smithi089 bash[21793]: audit 2024-07-15T06:56:10.058584+0000 mon.a (mon.0) 1088 : audit [INF] from='mgr.25016 ' entity='mgr.y' 2024-07-15T06:56:11.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:11 smithi089 bash[31372]: audit 2024-07-15T06:56:10.050965+0000 mon.a (mon.0) 1087 : audit [INF] from='mgr.25016 ' entity='mgr.y' 2024-07-15T06:56:11.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:11 smithi089 bash[31372]: audit 2024-07-15T06:56:10.058584+0000 mon.a (mon.0) 1088 : audit [INF] from='mgr.25016 ' entity='mgr.y' 2024-07-15T06:56:12.308 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:12 smithi134 bash[24068]: cluster 2024-07-15T06:56:10.954282+0000 mgr.y (mgr.25016) 10 : cluster [DBG] pgmap v7: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 16 op/s 2024-07-15T06:56:12.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:12 smithi134 bash[24068]: audit 2024-07-15T06:56:11.780349+0000 mon.a (mon.0) 1089 : audit [DBG] from='client.? 172.21.15.89:0/2008891039' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:56:12.369 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:12 smithi089 bash[21793]: cluster 2024-07-15T06:56:10.954282+0000 mgr.y (mgr.25016) 10 : cluster [DBG] pgmap v7: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 16 op/s 2024-07-15T06:56:12.369 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:12 smithi089 bash[21793]: audit 2024-07-15T06:56:11.780349+0000 mon.a (mon.0) 1089 : audit [DBG] from='client.? 172.21.15.89:0/2008891039' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:56:12.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:12 smithi089 bash[31372]: cluster 2024-07-15T06:56:10.954282+0000 mgr.y (mgr.25016) 10 : cluster [DBG] pgmap v7: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 16 op/s 2024-07-15T06:56:12.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:12 smithi089 bash[31372]: audit 2024-07-15T06:56:11.780349+0000 mon.a (mon.0) 1089 : audit [DBG] from='client.? 172.21.15.89:0/2008891039' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:56:12.803 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:56:12 smithi134 bash[58954]: ts=2024-07-15T06:56:12.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:56:13.803 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:56:13 smithi134 bash[58954]: ts=2024-07-15T06:56:13.509Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi134\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi134\", job=\"node\", machine=\"x86_64\", nodename=\"smithi134\", release=\"5.15.0-113-generic\", sysname=\"Linux\", version=\"#123-Ubuntu SMP Mon Jun 10 08:16:17 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", domainname=\"(none)\", instance=\"smithi134\", job=\"node\", machine=\"x86_64\", nodename=\"smithi134\", release=\"5.15.0-113-generic\", sysname=\"Linux\", version=\"#123-Ubuntu SMP Mon Jun 10 08:16:17 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:56:14.369 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:14 smithi089 bash[21793]: cluster 2024-07-15T06:56:12.954980+0000 mgr.y (mgr.25016) 11 : cluster [DBG] pgmap v8: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-07-15T06:56:14.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:14 smithi089 bash[31372]: cluster 2024-07-15T06:56:12.954980+0000 mgr.y (mgr.25016) 11 : cluster [DBG] pgmap v8: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-07-15T06:56:14.553 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:14 smithi134 bash[24068]: cluster 2024-07-15T06:56:12.954980+0000 mgr.y (mgr.25016) 11 : cluster [DBG] pgmap v8: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-07-15T06:56:15.369 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:15 smithi089 bash[21793]: audit 2024-07-15T06:56:14.158272+0000 mon.a (mon.0) 1090 : audit [DBG] from='client.? 172.21.15.89:0/1908225861' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:56:15.369 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:15 smithi089 bash[31372]: audit 2024-07-15T06:56:14.158272+0000 mon.a (mon.0) 1090 : audit [DBG] from='client.? 172.21.15.89:0/1908225861' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:56:15.553 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:15 smithi134 bash[24068]: audit 2024-07-15T06:56:14.158272+0000 mon.a (mon.0) 1090 : audit [DBG] from='client.? 172.21.15.89:0/1908225861' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:56:16.187 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:16 smithi089 bash[21793]: cluster 2024-07-15T06:56:14.956096+0000 mgr.y (mgr.25016) 12 : cluster [DBG] pgmap v9: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-07-15T06:56:16.188 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:16 smithi089 bash[31372]: cluster 2024-07-15T06:56:14.956096+0000 mgr.y (mgr.25016) 12 : cluster [DBG] pgmap v9: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-07-15T06:56:16.553 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:16 smithi134 bash[24068]: cluster 2024-07-15T06:56:14.956096+0000 mgr.y (mgr.25016) 12 : cluster [DBG] pgmap v9: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-07-15T06:56:17.369 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:17 smithi089 bash[21793]: audit 2024-07-15T06:56:16.525789+0000 mon.a (mon.0) 1091 : audit [DBG] from='client.? 172.21.15.89:0/2256577635' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:56:17.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:17 smithi089 bash[31372]: audit 2024-07-15T06:56:16.525789+0000 mon.a (mon.0) 1091 : audit [DBG] from='client.? 172.21.15.89:0/2256577635' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:56:17.553 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:17 smithi134 bash[24068]: audit 2024-07-15T06:56:16.525789+0000 mon.a (mon.0) 1091 : audit [DBG] from='client.? 172.21.15.89:0/2256577635' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:56:18.369 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:18 smithi089 bash[21793]: cluster 2024-07-15T06:56:16.956848+0000 mgr.y (mgr.25016) 13 : cluster [DBG] pgmap v10: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-07-15T06:56:18.369 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:18 smithi089 bash[31372]: cluster 2024-07-15T06:56:16.956848+0000 mgr.y (mgr.25016) 13 : cluster [DBG] pgmap v10: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-07-15T06:56:18.553 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:18 smithi134 bash[24068]: cluster 2024-07-15T06:56:16.956848+0000 mgr.y (mgr.25016) 13 : cluster [DBG] pgmap v10: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-07-15T06:56:19.369 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:19 smithi089 bash[21793]: audit 2024-07-15T06:56:18.172429+0000 mon.b (mon.1) 108 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:56:19.369 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:19 smithi089 bash[21793]: audit 2024-07-15T06:56:18.904403+0000 mon.a (mon.0) 1092 : audit [DBG] from='client.? 172.21.15.89:0/2690736819' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:56:19.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:19 smithi089 bash[31372]: audit 2024-07-15T06:56:18.172429+0000 mon.b (mon.1) 108 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:56:19.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:19 smithi089 bash[31372]: audit 2024-07-15T06:56:18.904403+0000 mon.a (mon.0) 1092 : audit [DBG] from='client.? 172.21.15.89:0/2690736819' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:56:19.553 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:19 smithi134 bash[24068]: audit 2024-07-15T06:56:18.172429+0000 mon.b (mon.1) 108 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:56:19.553 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:19 smithi134 bash[24068]: audit 2024-07-15T06:56:18.904403+0000 mon.a (mon.0) 1092 : audit [DBG] from='client.? 172.21.15.89:0/2690736819' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:56:20.053 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:56:19 smithi134 bash[52600]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:56:19] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-07-15T06:56:20.803 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:20 smithi134 bash[24068]: cluster 2024-07-15T06:56:18.957530+0000 mgr.y (mgr.25016) 14 : cluster [DBG] pgmap v11: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-07-15T06:56:20.803 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:20 smithi134 bash[24068]: audit 2024-07-15T06:56:19.503562+0000 mon.a (mon.0) 1093 : audit [INF] from='mgr.25016 ' entity='mgr.y' 2024-07-15T06:56:20.803 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:20 smithi134 bash[24068]: audit 2024-07-15T06:56:19.518488+0000 mon.a (mon.0) 1094 : audit [INF] from='mgr.25016 ' entity='mgr.y' 2024-07-15T06:56:20.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:20 smithi134 bash[24068]: audit 2024-07-15T06:56:19.524597+0000 mon.b (mon.1) 109 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:56:20.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:20 smithi134 bash[24068]: audit 2024-07-15T06:56:19.526857+0000 mon.b (mon.1) 110 : audit [INF] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:56:20.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:20 smithi134 bash[24068]: cephadm 2024-07-15T06:56:19.528271+0000 mgr.y (mgr.25016) 15 : cephadm [INF] Updating smithi089:/etc/ceph/ceph.conf 2024-07-15T06:56:20.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:20 smithi134 bash[24068]: cephadm 2024-07-15T06:56:19.528714+0000 mgr.y (mgr.25016) 16 : cephadm [INF] Updating smithi134:/etc/ceph/ceph.conf 2024-07-15T06:56:20.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:20 smithi134 bash[24068]: cephadm 2024-07-15T06:56:19.718333+0000 mgr.y (mgr.25016) 17 : cephadm [INF] Updating smithi089:/var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/config/ceph.conf 2024-07-15T06:56:20.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:20 smithi134 bash[24068]: cephadm 2024-07-15T06:56:19.721207+0000 mgr.y (mgr.25016) 18 : cephadm [INF] Updating smithi134:/var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/config/ceph.conf 2024-07-15T06:56:20.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:20 smithi134 bash[24068]: audit 2024-07-15T06:56:20.129042+0000 mon.a (mon.0) 1095 : audit [INF] from='mgr.25016 ' entity='mgr.y' 2024-07-15T06:56:20.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:20 smithi134 bash[24068]: audit 2024-07-15T06:56:20.139206+0000 mon.a (mon.0) 1096 : audit [INF] from='mgr.25016 ' entity='mgr.y' 2024-07-15T06:56:20.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:20 smithi134 bash[24068]: audit 2024-07-15T06:56:20.145722+0000 mon.a (mon.0) 1097 : audit [INF] from='mgr.25016 ' entity='mgr.y' 2024-07-15T06:56:20.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:20 smithi134 bash[24068]: audit 2024-07-15T06:56:20.157491+0000 mon.a (mon.0) 1098 : audit [INF] from='mgr.25016 ' entity='mgr.y' 2024-07-15T06:56:20.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:20 smithi134 bash[24068]: audit 2024-07-15T06:56:20.170011+0000 mon.a (mon.0) 1099 : audit [INF] from='mgr.25016 ' entity='mgr.y' 2024-07-15T06:56:20.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:20 smithi089 bash[21793]: cluster 2024-07-15T06:56:18.957530+0000 mgr.y (mgr.25016) 14 : cluster [DBG] pgmap v11: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-07-15T06:56:20.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:20 smithi089 bash[21793]: audit 2024-07-15T06:56:19.503562+0000 mon.a (mon.0) 1093 : audit [INF] from='mgr.25016 ' entity='mgr.y' 2024-07-15T06:56:20.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:20 smithi089 bash[21793]: audit 2024-07-15T06:56:19.518488+0000 mon.a (mon.0) 1094 : audit [INF] from='mgr.25016 ' entity='mgr.y' 2024-07-15T06:56:20.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:20 smithi089 bash[21793]: audit 2024-07-15T06:56:19.524597+0000 mon.b (mon.1) 109 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:56:20.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:20 smithi089 bash[21793]: audit 2024-07-15T06:56:19.526857+0000 mon.b (mon.1) 110 : audit [INF] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:56:20.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:20 smithi089 bash[21793]: cephadm 2024-07-15T06:56:19.528271+0000 mgr.y (mgr.25016) 15 : cephadm [INF] Updating smithi089:/etc/ceph/ceph.conf 2024-07-15T06:56:20.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:20 smithi089 bash[21793]: cephadm 2024-07-15T06:56:19.528714+0000 mgr.y (mgr.25016) 16 : cephadm [INF] Updating smithi134:/etc/ceph/ceph.conf 2024-07-15T06:56:20.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:20 smithi089 bash[21793]: cephadm 2024-07-15T06:56:19.718333+0000 mgr.y (mgr.25016) 17 : cephadm [INF] Updating smithi089:/var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/config/ceph.conf 2024-07-15T06:56:20.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:20 smithi089 bash[21793]: cephadm 2024-07-15T06:56:19.721207+0000 mgr.y (mgr.25016) 18 : cephadm [INF] Updating smithi134:/var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/config/ceph.conf 2024-07-15T06:56:20.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:20 smithi089 bash[21793]: audit 2024-07-15T06:56:20.129042+0000 mon.a (mon.0) 1095 : audit [INF] from='mgr.25016 ' entity='mgr.y' 2024-07-15T06:56:20.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:20 smithi089 bash[21793]: audit 2024-07-15T06:56:20.139206+0000 mon.a (mon.0) 1096 : audit [INF] from='mgr.25016 ' entity='mgr.y' 2024-07-15T06:56:20.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:20 smithi089 bash[21793]: audit 2024-07-15T06:56:20.145722+0000 mon.a (mon.0) 1097 : audit [INF] from='mgr.25016 ' entity='mgr.y' 2024-07-15T06:56:20.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:20 smithi089 bash[21793]: audit 2024-07-15T06:56:20.157491+0000 mon.a (mon.0) 1098 : audit [INF] from='mgr.25016 ' entity='mgr.y' 2024-07-15T06:56:20.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:20 smithi089 bash[21793]: audit 2024-07-15T06:56:20.170011+0000 mon.a (mon.0) 1099 : audit [INF] from='mgr.25016 ' entity='mgr.y' 2024-07-15T06:56:20.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:20 smithi089 bash[31372]: cluster 2024-07-15T06:56:18.957530+0000 mgr.y (mgr.25016) 14 : cluster [DBG] pgmap v11: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-07-15T06:56:20.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:20 smithi089 bash[31372]: audit 2024-07-15T06:56:19.503562+0000 mon.a (mon.0) 1093 : audit [INF] from='mgr.25016 ' entity='mgr.y' 2024-07-15T06:56:20.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:20 smithi089 bash[31372]: audit 2024-07-15T06:56:19.518488+0000 mon.a (mon.0) 1094 : audit [INF] from='mgr.25016 ' entity='mgr.y' 2024-07-15T06:56:20.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:20 smithi089 bash[31372]: audit 2024-07-15T06:56:19.524597+0000 mon.b (mon.1) 109 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:56:20.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:20 smithi089 bash[31372]: audit 2024-07-15T06:56:19.526857+0000 mon.b (mon.1) 110 : audit [INF] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:56:20.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:20 smithi089 bash[31372]: cephadm 2024-07-15T06:56:19.528271+0000 mgr.y (mgr.25016) 15 : cephadm [INF] Updating smithi089:/etc/ceph/ceph.conf 2024-07-15T06:56:20.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:20 smithi089 bash[31372]: cephadm 2024-07-15T06:56:19.528714+0000 mgr.y (mgr.25016) 16 : cephadm [INF] Updating smithi134:/etc/ceph/ceph.conf 2024-07-15T06:56:20.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:20 smithi089 bash[31372]: cephadm 2024-07-15T06:56:19.718333+0000 mgr.y (mgr.25016) 17 : cephadm [INF] Updating smithi089:/var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/config/ceph.conf 2024-07-15T06:56:20.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:20 smithi089 bash[31372]: cephadm 2024-07-15T06:56:19.721207+0000 mgr.y (mgr.25016) 18 : cephadm [INF] Updating smithi134:/var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/config/ceph.conf 2024-07-15T06:56:20.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:20 smithi089 bash[31372]: audit 2024-07-15T06:56:20.129042+0000 mon.a (mon.0) 1095 : audit [INF] from='mgr.25016 ' entity='mgr.y' 2024-07-15T06:56:20.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:20 smithi089 bash[31372]: audit 2024-07-15T06:56:20.139206+0000 mon.a (mon.0) 1096 : audit [INF] from='mgr.25016 ' entity='mgr.y' 2024-07-15T06:56:20.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:20 smithi089 bash[31372]: audit 2024-07-15T06:56:20.145722+0000 mon.a (mon.0) 1097 : audit [INF] from='mgr.25016 ' entity='mgr.y' 2024-07-15T06:56:20.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:20 smithi089 bash[31372]: audit 2024-07-15T06:56:20.157491+0000 mon.a (mon.0) 1098 : audit [INF] from='mgr.25016 ' entity='mgr.y' 2024-07-15T06:56:20.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:20 smithi089 bash[31372]: audit 2024-07-15T06:56:20.170011+0000 mon.a (mon.0) 1099 : audit [INF] from='mgr.25016 ' entity='mgr.y' 2024-07-15T06:56:21.803 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:21 smithi134 bash[24068]: cephadm 2024-07-15T06:56:19.848100+0000 mgr.y (mgr.25016) 19 : cephadm [INF] Updating smithi089:/etc/ceph/ceph.client.admin.keyring 2024-07-15T06:56:21.803 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:21 smithi134 bash[24068]: cephadm 2024-07-15T06:56:19.861472+0000 mgr.y (mgr.25016) 20 : cephadm [INF] Updating smithi134:/etc/ceph/ceph.client.admin.keyring 2024-07-15T06:56:21.803 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:21 smithi134 bash[24068]: cephadm 2024-07-15T06:56:19.982908+0000 mgr.y (mgr.25016) 21 : cephadm [INF] Updating smithi089:/var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/config/ceph.client.admin.keyring 2024-07-15T06:56:21.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:21 smithi134 bash[24068]: cephadm 2024-07-15T06:56:19.998776+0000 mgr.y (mgr.25016) 22 : cephadm [INF] Updating smithi134:/var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/config/ceph.client.admin.keyring 2024-07-15T06:56:21.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:21 smithi134 bash[24068]: cephadm 2024-07-15T06:56:20.214843+0000 mgr.y (mgr.25016) 23 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2024-07-15T06:56:21.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:21 smithi134 bash[24068]: cephadm 2024-07-15T06:56:20.510134+0000 mgr.y (mgr.25016) 24 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi134 2024-07-15T06:56:21.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:21 smithi134 bash[24068]: audit 2024-07-15T06:56:21.272745+0000 mon.a (mon.0) 1100 : audit [DBG] from='client.? 172.21.15.89:0/2812331157' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:56:21.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:21 smithi089 bash[21793]: cephadm 2024-07-15T06:56:19.848100+0000 mgr.y (mgr.25016) 19 : cephadm [INF] Updating smithi089:/etc/ceph/ceph.client.admin.keyring 2024-07-15T06:56:21.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:21 smithi089 bash[21793]: cephadm 2024-07-15T06:56:19.861472+0000 mgr.y (mgr.25016) 20 : cephadm [INF] Updating smithi134:/etc/ceph/ceph.client.admin.keyring 2024-07-15T06:56:21.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:21 smithi089 bash[21793]: cephadm 2024-07-15T06:56:19.982908+0000 mgr.y (mgr.25016) 21 : cephadm [INF] Updating smithi089:/var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/config/ceph.client.admin.keyring 2024-07-15T06:56:21.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:21 smithi089 bash[21793]: cephadm 2024-07-15T06:56:19.998776+0000 mgr.y (mgr.25016) 22 : cephadm [INF] Updating smithi134:/var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/config/ceph.client.admin.keyring 2024-07-15T06:56:21.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:21 smithi089 bash[21793]: cephadm 2024-07-15T06:56:20.214843+0000 mgr.y (mgr.25016) 23 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2024-07-15T06:56:21.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:21 smithi089 bash[21793]: cephadm 2024-07-15T06:56:20.510134+0000 mgr.y (mgr.25016) 24 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi134 2024-07-15T06:56:21.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:21 smithi089 bash[21793]: audit 2024-07-15T06:56:21.272745+0000 mon.a (mon.0) 1100 : audit [DBG] from='client.? 172.21.15.89:0/2812331157' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:56:21.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:21 smithi089 bash[31372]: cephadm 2024-07-15T06:56:19.848100+0000 mgr.y (mgr.25016) 19 : cephadm [INF] Updating smithi089:/etc/ceph/ceph.client.admin.keyring 2024-07-15T06:56:21.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:21 smithi089 bash[31372]: cephadm 2024-07-15T06:56:19.861472+0000 mgr.y (mgr.25016) 20 : cephadm [INF] Updating smithi134:/etc/ceph/ceph.client.admin.keyring 2024-07-15T06:56:21.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:21 smithi089 bash[31372]: cephadm 2024-07-15T06:56:19.982908+0000 mgr.y (mgr.25016) 21 : cephadm [INF] Updating smithi089:/var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/config/ceph.client.admin.keyring 2024-07-15T06:56:21.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:21 smithi089 bash[31372]: cephadm 2024-07-15T06:56:19.998776+0000 mgr.y (mgr.25016) 22 : cephadm [INF] Updating smithi134:/var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/config/ceph.client.admin.keyring 2024-07-15T06:56:21.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:21 smithi089 bash[31372]: cephadm 2024-07-15T06:56:20.214843+0000 mgr.y (mgr.25016) 23 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2024-07-15T06:56:21.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:21 smithi089 bash[31372]: cephadm 2024-07-15T06:56:20.510134+0000 mgr.y (mgr.25016) 24 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi134 2024-07-15T06:56:21.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:21 smithi089 bash[31372]: audit 2024-07-15T06:56:21.272745+0000 mon.a (mon.0) 1100 : audit [DBG] from='client.? 172.21.15.89:0/2812331157' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:56:22.803 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:22 smithi134 bash[24068]: cluster 2024-07-15T06:56:20.958331+0000 mgr.y (mgr.25016) 25 : cluster [DBG] pgmap v12: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-07-15T06:56:22.804 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:56:22 smithi134 bash[58954]: ts=2024-07-15T06:56:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:56:22.869 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:22 smithi089 bash[21793]: cluster 2024-07-15T06:56:20.958331+0000 mgr.y (mgr.25016) 25 : cluster [DBG] pgmap v12: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-07-15T06:56:22.869 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:22 smithi089 bash[31372]: cluster 2024-07-15T06:56:20.958331+0000 mgr.y (mgr.25016) 25 : cluster [DBG] pgmap v12: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-07-15T06:56:23.553 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:56:23 smithi134 bash[58954]: ts=2024-07-15T06:56:23.508Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi134\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi134\", job=\"node\", machine=\"x86_64\", nodename=\"smithi134\", release=\"5.15.0-113-generic\", sysname=\"Linux\", version=\"#123-Ubuntu SMP Mon Jun 10 08:16:17 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", domainname=\"(none)\", instance=\"smithi134\", job=\"node\", machine=\"x86_64\", nodename=\"smithi134\", release=\"5.15.0-113-generic\", sysname=\"Linux\", version=\"#123-Ubuntu SMP Mon Jun 10 08:16:17 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:56:24.792 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:56:24 smithi134 systemd[1]: Stopping Ceph prometheus.a for 75ba02bc-4274-11ef-bcad-c7b262605968... 2024-07-15T06:56:24.793 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:24 smithi134 bash[24068]: cluster 2024-07-15T06:56:22.958971+0000 mgr.y (mgr.25016) 26 : cluster [DBG] pgmap v13: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:56:24.793 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:24 smithi134 bash[24068]: audit 2024-07-15T06:56:23.634961+0000 mon.c (mon.2) 60 : audit [DBG] from='client.? 172.21.15.89:0/3968819325' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:56:24.869 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:24 smithi089 bash[21793]: cluster 2024-07-15T06:56:22.958971+0000 mgr.y (mgr.25016) 26 : cluster [DBG] pgmap v13: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:56:24.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:24 smithi089 bash[21793]: audit 2024-07-15T06:56:23.634961+0000 mon.c (mon.2) 60 : audit [DBG] from='client.? 172.21.15.89:0/3968819325' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:56:24.870 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:24 smithi089 bash[31372]: cluster 2024-07-15T06:56:22.958971+0000 mgr.y (mgr.25016) 26 : cluster [DBG] pgmap v13: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:56:24.870 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:24 smithi089 bash[31372]: audit 2024-07-15T06:56:23.634961+0000 mon.c (mon.2) 60 : audit [DBG] from='client.? 172.21.15.89:0/3968819325' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:56:25.053 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:56:24 smithi134 bash[58954]: ts=2024-07-15T06:56:24.799Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-07-15T06:56:25.054 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:56:24 smithi134 bash[58954]: ts=2024-07-15T06:56:24.799Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-07-15T06:56:25.054 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:56:24 smithi134 bash[58954]: ts=2024-07-15T06:56:24.799Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-07-15T06:56:25.054 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:56:24 smithi134 bash[58954]: ts=2024-07-15T06:56:24.799Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-07-15T06:56:25.054 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:56:24 smithi134 bash[58954]: ts=2024-07-15T06:56:24.799Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-07-15T06:56:25.054 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:56:24 smithi134 bash[58954]: ts=2024-07-15T06:56:24.799Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-07-15T06:56:25.054 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:56:24 smithi134 bash[58954]: ts=2024-07-15T06:56:24.799Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-07-15T06:56:25.054 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:56:24 smithi134 bash[58954]: ts=2024-07-15T06:56:24.799Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-07-15T06:56:25.054 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:56:24 smithi134 bash[58954]: ts=2024-07-15T06:56:24.801Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-07-15T06:56:25.054 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:56:24 smithi134 bash[58954]: ts=2024-07-15T06:56:24.802Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-07-15T06:56:25.054 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:56:24 smithi134 bash[58954]: ts=2024-07-15T06:56:24.802Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-07-15T06:56:25.054 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:56:24 smithi134 bash[58954]: ts=2024-07-15T06:56:24.802Z caller=main.go:1147 level=info msg="See you next time!" 2024-07-15T06:56:25.553 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:56:25 smithi134 bash[60638]: ceph-75ba02bc-4274-11ef-bcad-c7b262605968-prometheus-a 2024-07-15T06:56:25.553 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:56:25 smithi134 bash[60700]: Error response from daemon: No such container: ceph-75ba02bc-4274-11ef-bcad-c7b262605968-prometheus-a 2024-07-15T06:56:25.553 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:56:25 smithi134 systemd[1]: ceph-75ba02bc-4274-11ef-bcad-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-07-15T06:56:25.553 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:56:25 smithi134 systemd[1]: Stopped Ceph prometheus.a for 75ba02bc-4274-11ef-bcad-c7b262605968. 2024-07-15T06:56:25.553 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:56:25 smithi134 systemd[1]: Started Ceph prometheus.a for 75ba02bc-4274-11ef-bcad-c7b262605968. 2024-07-15T06:56:26.803 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:26 smithi134 bash[24068]: cluster 2024-07-15T06:56:24.960119+0000 mgr.y (mgr.25016) 27 : cluster [DBG] pgmap v14: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:56:26.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:26 smithi134 bash[24068]: audit 2024-07-15T06:56:25.545751+0000 mon.a (mon.0) 1101 : audit [INF] from='mgr.25016 ' entity='mgr.y' 2024-07-15T06:56:26.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:26 smithi134 bash[24068]: audit 2024-07-15T06:56:25.560067+0000 mon.a (mon.0) 1102 : audit [INF] from='mgr.25016 ' entity='mgr.y' 2024-07-15T06:56:26.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:26 smithi134 bash[24068]: audit 2024-07-15T06:56:25.568111+0000 mgr.y (mgr.25016) 28 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-07-15T06:56:26.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:26 smithi134 bash[24068]: audit 2024-07-15T06:56:25.568303+0000 mon.b (mon.1) 111 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-07-15T06:56:26.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:26 smithi134 bash[24068]: audit 2024-07-15T06:56:25.610835+0000 mon.b (mon.1) 112 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:56:26.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:26 smithi134 bash[24068]: audit 2024-07-15T06:56:25.985469+0000 mon.c (mon.2) 61 : audit [DBG] from='client.? 172.21.15.89:0/2932259376' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:56:26.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:26 smithi089 bash[21793]: cluster 2024-07-15T06:56:24.960119+0000 mgr.y (mgr.25016) 27 : cluster [DBG] pgmap v14: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:56:26.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:26 smithi089 bash[21793]: audit 2024-07-15T06:56:25.545751+0000 mon.a (mon.0) 1101 : audit [INF] from='mgr.25016 ' entity='mgr.y' 2024-07-15T06:56:26.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:26 smithi089 bash[21793]: audit 2024-07-15T06:56:25.560067+0000 mon.a (mon.0) 1102 : audit [INF] from='mgr.25016 ' entity='mgr.y' 2024-07-15T06:56:26.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:26 smithi089 bash[21793]: audit 2024-07-15T06:56:25.568111+0000 mgr.y (mgr.25016) 28 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-07-15T06:56:26.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:26 smithi089 bash[21793]: audit 2024-07-15T06:56:25.568303+0000 mon.b (mon.1) 111 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-07-15T06:56:26.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:26 smithi089 bash[21793]: audit 2024-07-15T06:56:25.610835+0000 mon.b (mon.1) 112 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:56:26.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:26 smithi089 bash[21793]: audit 2024-07-15T06:56:25.985469+0000 mon.c (mon.2) 61 : audit [DBG] from='client.? 172.21.15.89:0/2932259376' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:56:26.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:26 smithi089 bash[31372]: cluster 2024-07-15T06:56:24.960119+0000 mgr.y (mgr.25016) 27 : cluster [DBG] pgmap v14: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:56:26.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:26 smithi089 bash[31372]: audit 2024-07-15T06:56:25.545751+0000 mon.a (mon.0) 1101 : audit [INF] from='mgr.25016 ' entity='mgr.y' 2024-07-15T06:56:26.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:26 smithi089 bash[31372]: audit 2024-07-15T06:56:25.560067+0000 mon.a (mon.0) 1102 : audit [INF] from='mgr.25016 ' entity='mgr.y' 2024-07-15T06:56:26.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:26 smithi089 bash[31372]: audit 2024-07-15T06:56:25.568111+0000 mgr.y (mgr.25016) 28 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-07-15T06:56:26.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:26 smithi089 bash[31372]: audit 2024-07-15T06:56:25.568303+0000 mon.b (mon.1) 111 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-07-15T06:56:26.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:26 smithi089 bash[31372]: audit 2024-07-15T06:56:25.610835+0000 mon.b (mon.1) 112 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:56:26.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:26 smithi089 bash[31372]: audit 2024-07-15T06:56:25.985469+0000 mon.c (mon.2) 61 : audit [DBG] from='client.? 172.21.15.89:0/2932259376' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:56:27.795 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:56:27 smithi134 bash[60729]: ts=2024-07-15T06:56:27.658Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-07-15T06:56:27.795 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:56:27 smithi134 bash[60729]: ts=2024-07-15T06:56:27.658Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-07-15T06:56:27.795 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:56:27 smithi134 bash[60729]: ts=2024-07-15T06:56:27.658Z caller=main.go:570 level=info host_details="(Linux 5.15.0-113-generic #123-Ubuntu SMP Mon Jun 10 08:16:17 UTC 2024 x86_64 smithi134 (none))" 2024-07-15T06:56:27.795 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:56:27 smithi134 bash[60729]: ts=2024-07-15T06:56:27.658Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-07-15T06:56:27.796 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:56:27 smithi134 bash[60729]: ts=2024-07-15T06:56:27.658Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-07-15T06:56:27.796 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:56:27 smithi134 bash[60729]: ts=2024-07-15T06:56:27.659Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-07-15T06:56:27.796 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:56:27 smithi134 bash[60729]: ts=2024-07-15T06:56:27.660Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-07-15T06:56:27.796 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:56:27 smithi134 bash[60729]: ts=2024-07-15T06:56:27.660Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-07-15T06:56:27.796 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:56:27 smithi134 bash[60729]: ts=2024-07-15T06:56:27.660Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-07-15T06:56:27.796 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:56:27 smithi134 bash[60729]: ts=2024-07-15T06:56:27.662Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-07-15T06:56:27.796 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:56:27 smithi134 bash[60729]: ts=2024-07-15T06:56:27.662Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.912µs 2024-07-15T06:56:27.796 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:56:27 smithi134 bash[60729]: ts=2024-07-15T06:56:27.663Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-07-15T06:56:27.796 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:56:27 smithi134 bash[60729]: ts=2024-07-15T06:56:27.664Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=3 2024-07-15T06:56:27.796 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:56:27 smithi134 bash[60729]: ts=2024-07-15T06:56:27.689Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=3 2024-07-15T06:56:27.796 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:56:27 smithi134 bash[60729]: ts=2024-07-15T06:56:27.697Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=3 2024-07-15T06:56:27.796 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:56:27 smithi134 bash[60729]: ts=2024-07-15T06:56:27.700Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=3 2024-07-15T06:56:27.796 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:56:27 smithi134 bash[60729]: ts=2024-07-15T06:56:27.700Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=63.468µs wal_replay_duration=37.211669ms wbl_replay_duration=133ns total_replay_duration=37.314906ms 2024-07-15T06:56:27.796 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:56:27 smithi134 bash[60729]: ts=2024-07-15T06:56:27.703Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-07-15T06:56:27.797 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:56:27 smithi134 bash[60729]: ts=2024-07-15T06:56:27.703Z caller=main.go:1029 level=info msg="TSDB started" 2024-07-15T06:56:27.797 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:56:27 smithi134 bash[60729]: ts=2024-07-15T06:56:27.703Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-07-15T06:56:27.797 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:56:27 smithi134 bash[60729]: ts=2024-07-15T06:56:27.725Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=21.136905ms db_storage=896ns remote_storage=1.138µs web_handler=462ns query_engine=882ns scrape=318.795µs scrape_sd=120.413µs notify=11.291µs notify_sd=18.661µs rules=20.174169ms tracing=6.261µs 2024-07-15T06:56:27.797 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:56:27 smithi134 bash[60729]: ts=2024-07-15T06:56:27.725Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-07-15T06:56:27.797 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:56:27 smithi134 bash[60729]: ts=2024-07-15T06:56:27.725Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-07-15T06:56:28.803 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:28 smithi134 bash[24068]: cluster 2024-07-15T06:56:26.960792+0000 mgr.y (mgr.25016) 29 : cluster [DBG] pgmap v15: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:56:28.803 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:28 smithi134 bash[24068]: audit 2024-07-15T06:56:28.357208+0000 mon.c (mon.2) 62 : audit [DBG] from='client.? 172.21.15.89:0/985820048' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:56:28.803 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:28 smithi134 bash[24068]: audit 2024-07-15T06:56:28.457153+0000 mon.a (mon.0) 1103 : audit [INF] from='mgr.25016 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-07-15T06:56:28.803 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:28 smithi134 bash[24068]: audit 2024-07-15T06:56:28.457603+0000 mon.b (mon.1) 113 : audit [INF] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-07-15T06:56:28.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:28 smithi134 bash[24068]: audit 2024-07-15T06:56:28.460684+0000 mon.b (mon.1) 114 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-07-15T06:56:28.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:28 smithi134 bash[24068]: audit 2024-07-15T06:56:28.462981+0000 mon.b (mon.1) 115 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:56:28.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:28 smithi089 bash[21793]: cluster 2024-07-15T06:56:26.960792+0000 mgr.y (mgr.25016) 29 : cluster [DBG] pgmap v15: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:56:28.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:28 smithi089 bash[21793]: audit 2024-07-15T06:56:28.357208+0000 mon.c (mon.2) 62 : audit [DBG] from='client.? 172.21.15.89:0/985820048' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:56:28.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:28 smithi089 bash[21793]: audit 2024-07-15T06:56:28.457153+0000 mon.a (mon.0) 1103 : audit [INF] from='mgr.25016 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-07-15T06:56:28.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:28 smithi089 bash[21793]: audit 2024-07-15T06:56:28.457603+0000 mon.b (mon.1) 113 : audit [INF] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-07-15T06:56:28.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:28 smithi089 bash[21793]: audit 2024-07-15T06:56:28.460684+0000 mon.b (mon.1) 114 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-07-15T06:56:28.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:28 smithi089 bash[21793]: audit 2024-07-15T06:56:28.462981+0000 mon.b (mon.1) 115 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:56:28.870 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:28 smithi089 bash[31372]: cluster 2024-07-15T06:56:26.960792+0000 mgr.y (mgr.25016) 29 : cluster [DBG] pgmap v15: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:56:28.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:28 smithi089 bash[31372]: audit 2024-07-15T06:56:28.357208+0000 mon.c (mon.2) 62 : audit [DBG] from='client.? 172.21.15.89:0/985820048' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:56:28.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:28 smithi089 bash[31372]: audit 2024-07-15T06:56:28.457153+0000 mon.a (mon.0) 1103 : audit [INF] from='mgr.25016 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-07-15T06:56:28.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:28 smithi089 bash[31372]: audit 2024-07-15T06:56:28.457603+0000 mon.b (mon.1) 113 : audit [INF] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-07-15T06:56:28.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:28 smithi089 bash[31372]: audit 2024-07-15T06:56:28.460684+0000 mon.b (mon.1) 114 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-07-15T06:56:28.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:28 smithi089 bash[31372]: audit 2024-07-15T06:56:28.462981+0000 mon.b (mon.1) 115 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:56:29.803 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:29 smithi134 bash[24068]: cephadm 2024-07-15T06:56:28.455413+0000 mgr.y (mgr.25016) 30 : cephadm [INF] Upgrade: Updating mgr.x 2024-07-15T06:56:29.803 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:29 smithi134 bash[24068]: cephadm 2024-07-15T06:56:28.463599+0000 mgr.y (mgr.25016) 31 : cephadm [INF] Deploying daemon mgr.x on smithi134 2024-07-15T06:56:29.869 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:29 smithi089 bash[21793]: cephadm 2024-07-15T06:56:28.455413+0000 mgr.y (mgr.25016) 30 : cephadm [INF] Upgrade: Updating mgr.x 2024-07-15T06:56:29.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:29 smithi089 bash[21793]: cephadm 2024-07-15T06:56:28.463599+0000 mgr.y (mgr.25016) 31 : cephadm [INF] Deploying daemon mgr.x on smithi134 2024-07-15T06:56:29.870 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:29 smithi089 bash[31372]: cephadm 2024-07-15T06:56:28.455413+0000 mgr.y (mgr.25016) 30 : cephadm [INF] Upgrade: Updating mgr.x 2024-07-15T06:56:29.870 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:29 smithi089 bash[31372]: cephadm 2024-07-15T06:56:28.463599+0000 mgr.y (mgr.25016) 31 : cephadm [INF] Deploying daemon mgr.x on smithi134 2024-07-15T06:56:30.803 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:56:30 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:56:30.803 INFO:journalctl@ceph.osd.4.smithi134.stdout:Jul 15 06:56:30 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:56:30.804 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:56:30 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:56:30.804 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:56:30 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:56:30.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:30 smithi134 bash[24068]: cluster 2024-07-15T06:56:28.961466+0000 mgr.y (mgr.25016) 32 : cluster [DBG] pgmap v16: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:56:30.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:30 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:56:30.805 INFO:journalctl@ceph.osd.5.smithi134.stdout:Jul 15 06:56:30 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:56:30.805 INFO:journalctl@ceph.osd.6.smithi134.stdout:Jul 15 06:56:30 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:56:30.805 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 06:56:30 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:56:30.806 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:56:30 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:56:30.869 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:30 smithi089 bash[21793]: cluster 2024-07-15T06:56:28.961466+0000 mgr.y (mgr.25016) 32 : cluster [DBG] pgmap v16: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:56:30.870 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:30 smithi089 bash[31372]: cluster 2024-07-15T06:56:28.961466+0000 mgr.y (mgr.25016) 32 : cluster [DBG] pgmap v16: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:56:31.267 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:56:30 smithi134 systemd[1]: Stopping Ceph mgr.x for 75ba02bc-4274-11ef-bcad-c7b262605968... 2024-07-15T06:56:31.553 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:56:31 smithi134 bash[61012]: ceph-75ba02bc-4274-11ef-bcad-c7b262605968-mgr-x 2024-07-15T06:56:31.869 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:31 smithi089 bash[21793]: audit 2024-07-15T06:56:30.730126+0000 mon.a (mon.0) 1104 : audit [DBG] from='client.? 172.21.15.89:0/2846647066' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:56:31.870 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:31 smithi089 bash[31372]: audit 2024-07-15T06:56:30.730126+0000 mon.a (mon.0) 1104 : audit [DBG] from='client.? 172.21.15.89:0/2846647066' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:56:31.894 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:31 smithi134 bash[24068]: audit 2024-07-15T06:56:30.730126+0000 mon.a (mon.0) 1104 : audit [DBG] from='client.? 172.21.15.89:0/2846647066' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:56:31.894 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:56:31 smithi134 systemd[1]: ceph-75ba02bc-4274-11ef-bcad-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-07-15T06:56:31.894 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:56:31 smithi134 bash[61072]: Error response from daemon: No such container: ceph-75ba02bc-4274-11ef-bcad-c7b262605968-mgr-x 2024-07-15T06:56:31.894 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:56:31 smithi134 systemd[1]: ceph-75ba02bc-4274-11ef-bcad-c7b262605968@mgr.x.service: Failed with result 'exit-code'. 2024-07-15T06:56:31.894 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:56:31 smithi134 systemd[1]: Stopped Ceph mgr.x for 75ba02bc-4274-11ef-bcad-c7b262605968. 2024-07-15T06:56:32.303 INFO:journalctl@ceph.osd.4.smithi134.stdout:Jul 15 06:56:31 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:56:32.303 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:31 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:56:32.304 INFO:journalctl@ceph.osd.5.smithi134.stdout:Jul 15 06:56:31 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:56:32.304 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:56:31 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:56:32.304 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:56:31 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:56:32.304 INFO:journalctl@ceph.osd.6.smithi134.stdout:Jul 15 06:56:31 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:56:32.305 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:56:31 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:56:32.305 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:56:32 smithi134 systemd[1]: Started Ceph mgr.x for 75ba02bc-4274-11ef-bcad-c7b262605968. 2024-07-15T06:56:32.305 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:56:31 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:56:32.305 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 06:56:31 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:56:32.563 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:56:32 smithi134 bash[60729]: ts=2024-07-15T06:56:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:56:32.869 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:32 smithi089 bash[21793]: cluster 2024-07-15T06:56:30.962556+0000 mgr.y (mgr.25016) 33 : cluster [DBG] pgmap v17: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:56:32.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:32 smithi089 bash[21793]: audit 2024-07-15T06:56:32.062961+0000 mon.a (mon.0) 1105 : audit [INF] from='mgr.25016 ' entity='mgr.y' 2024-07-15T06:56:32.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:32 smithi089 bash[21793]: audit 2024-07-15T06:56:32.074719+0000 mon.a (mon.0) 1106 : audit [INF] from='mgr.25016 ' entity='mgr.y' 2024-07-15T06:56:32.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:32 smithi089 bash[21793]: audit 2024-07-15T06:56:32.078907+0000 mon.b (mon.1) 116 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:56:32.870 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:32 smithi089 bash[31372]: cluster 2024-07-15T06:56:30.962556+0000 mgr.y (mgr.25016) 33 : cluster [DBG] pgmap v17: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:56:32.870 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:32 smithi089 bash[31372]: audit 2024-07-15T06:56:32.062961+0000 mon.a (mon.0) 1105 : audit [INF] from='mgr.25016 ' entity='mgr.y' 2024-07-15T06:56:32.870 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:32 smithi089 bash[31372]: audit 2024-07-15T06:56:32.074719+0000 mon.a (mon.0) 1106 : audit [INF] from='mgr.25016 ' entity='mgr.y' 2024-07-15T06:56:32.870 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:32 smithi089 bash[31372]: audit 2024-07-15T06:56:32.078907+0000 mon.b (mon.1) 116 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:56:33.053 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:32 smithi134 bash[24068]: cluster 2024-07-15T06:56:30.962556+0000 mgr.y (mgr.25016) 33 : cluster [DBG] pgmap v17: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:56:33.053 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:32 smithi134 bash[24068]: audit 2024-07-15T06:56:32.062961+0000 mon.a (mon.0) 1105 : audit [INF] from='mgr.25016 ' entity='mgr.y' 2024-07-15T06:56:33.053 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:32 smithi134 bash[24068]: audit 2024-07-15T06:56:32.074719+0000 mon.a (mon.0) 1106 : audit [INF] from='mgr.25016 ' entity='mgr.y' 2024-07-15T06:56:33.053 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:32 smithi134 bash[24068]: audit 2024-07-15T06:56:32.078907+0000 mon.b (mon.1) 116 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:56:33.803 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:56:33 smithi134 bash[60729]: ts=2024-07-15T06:56:33.506Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi134\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi134\", job=\"node\", machine=\"x86_64\", nodename=\"smithi134\", release=\"5.15.0-113-generic\", sysname=\"Linux\", version=\"#123-Ubuntu SMP Mon Jun 10 08:16:17 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", domainname=\"(none)\", instance=\"smithi134\", job=\"node\", machine=\"x86_64\", nodename=\"smithi134\", release=\"5.15.0-113-generic\", sysname=\"Linux\", version=\"#123-Ubuntu SMP Mon Jun 10 08:16:17 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:56:33.803 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:33 smithi134 bash[24068]: audit 2024-07-15T06:56:33.107199+0000 mon.c (mon.2) 63 : audit [DBG] from='client.? 172.21.15.89:0/2038918363' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:56:33.803 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:33 smithi134 bash[24068]: audit 2024-07-15T06:56:33.172786+0000 mon.b (mon.1) 117 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:56:33.804 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:56:33 smithi134 bash[61128]: debug 2024-07-15T06:56:33.442+0000 7f861cec8640 1 -- 172.21.15.134:0/3489494057 <== mon.1 v2:172.21.15.134:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55cb60224d00 con 0x55cb60217800 2024-07-15T06:56:33.804 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:56:33 smithi134 bash[61128]: debug 2024-07-15T06:56:33.558+0000 7f861f731040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-07-15T06:56:33.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:33 smithi089 bash[21793]: audit 2024-07-15T06:56:33.107199+0000 mon.c (mon.2) 63 : audit [DBG] from='client.? 172.21.15.89:0/2038918363' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:56:33.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:33 smithi089 bash[21793]: audit 2024-07-15T06:56:33.172786+0000 mon.b (mon.1) 117 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:56:33.870 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:33 smithi089 bash[31372]: audit 2024-07-15T06:56:33.107199+0000 mon.c (mon.2) 63 : audit [DBG] from='client.? 172.21.15.89:0/2038918363' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:56:33.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:33 smithi089 bash[31372]: audit 2024-07-15T06:56:33.172786+0000 mon.b (mon.1) 117 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:56:34.574 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:56:34 smithi089 bash[72894]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:56:34] "GET /metrics HTTP/1.1" 200 33797 "" "Prometheus/2.43.0" 2024-07-15T06:56:34.789 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:34 smithi134 bash[24068]: cluster 2024-07-15T06:56:32.963178+0000 mgr.y (mgr.25016) 34 : cluster [DBG] pgmap v18: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:56:34.789 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:56:34 smithi134 bash[60729]: ts=2024-07-15T06:56:34.453Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:56:34.789 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:56:34 smithi134 bash[60729]: ts=2024-07-15T06:56:34.454Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:56:34.869 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:34 smithi089 bash[21793]: cluster 2024-07-15T06:56:32.963178+0000 mgr.y (mgr.25016) 34 : cluster [DBG] pgmap v18: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:56:34.870 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:34 smithi089 bash[31372]: cluster 2024-07-15T06:56:32.963178+0000 mgr.y (mgr.25016) 34 : cluster [DBG] pgmap v18: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:56:35.053 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:56:34 smithi134 bash[61128]: debug 2024-07-15T06:56:34.790+0000 7f861f731040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-07-15T06:56:35.053 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:56:34 smithi134 bash[61128]: debug 2024-07-15T06:56:34.858+0000 7f861f731040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-07-15T06:56:35.053 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:56:35 smithi134 bash[61128]: debug 2024-07-15T06:56:35.010+0000 7f861f731040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-07-15T06:56:35.869 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:35 smithi089 bash[21793]: audit 2024-07-15T06:56:35.486077+0000 mon.c (mon.2) 64 : audit [DBG] from='client.? 172.21.15.89:0/718823002' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:56:35.870 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:35 smithi089 bash[31372]: audit 2024-07-15T06:56:35.486077+0000 mon.c (mon.2) 64 : audit [DBG] from='client.? 172.21.15.89:0/718823002' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:56:35.998 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:35 smithi134 bash[24068]: audit 2024-07-15T06:56:35.486077+0000 mon.c (mon.2) 64 : audit [DBG] from='client.? 172.21.15.89:0/718823002' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:56:35.998 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:56:35 smithi134 bash[61128]: debug 2024-07-15T06:56:35.670+0000 7f861f731040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-07-15T06:56:35.998 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:56:35 smithi134 bash[61128]: debug 2024-07-15T06:56:35.738+0000 7f861f731040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-07-15T06:56:35.999 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:56:35 smithi134 bash[61128]: debug 2024-07-15T06:56:35.810+0000 7f861f731040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-07-15T06:56:35.999 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:56:36 smithi134 bash[61128]: debug 2024-07-15T06:56:35.998+0000 7f861f731040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-07-15T06:56:36.303 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:56:36 smithi134 bash[61128]: debug 2024-07-15T06:56:36.078+0000 7f861f731040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-07-15T06:56:36.303 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:56:36 smithi134 bash[61128]: debug 2024-07-15T06:56:36.146+0000 7f861f731040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-07-15T06:56:36.685 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:56:36 smithi134 bash[61128]: debug 2024-07-15T06:56:36.410+0000 7f861f731040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-07-15T06:56:36.685 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:56:36 smithi134 bash[61128]: debug 2024-07-15T06:56:36.478+0000 7f861f731040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-07-15T06:56:36.685 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:36 smithi134 bash[24068]: cluster 2024-07-15T06:56:34.964346+0000 mgr.y (mgr.25016) 35 : cluster [DBG] pgmap v19: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:56:36.869 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:36 smithi089 bash[21793]: cluster 2024-07-15T06:56:34.964346+0000 mgr.y (mgr.25016) 35 : cluster [DBG] pgmap v19: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:56:36.869 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:36 smithi089 bash[31372]: cluster 2024-07-15T06:56:34.964346+0000 mgr.y (mgr.25016) 35 : cluster [DBG] pgmap v19: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:56:37.020 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:56:36 smithi134 bash[61128]: debug 2024-07-15T06:56:36.686+0000 7f861f731040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-07-15T06:56:37.020 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:56:36 smithi134 bash[61128]: debug 2024-07-15T06:56:36.766+0000 7f861f731040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-07-15T06:56:37.303 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:56:37 smithi134 bash[61128]: debug 2024-07-15T06:56:37.022+0000 7f861f731040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-07-15T06:56:37.303 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:56:37 smithi134 bash[61128]: debug 2024-07-15T06:56:37.202+0000 7f861f731040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-07-15T06:56:37.638 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:56:37 smithi134 bash[61128]: debug 2024-07-15T06:56:37.374+0000 7f861f731040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-07-15T06:56:37.639 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:56:37 smithi134 bash[61128]: debug 2024-07-15T06:56:37.458+0000 7f861f731040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-07-15T06:56:38.053 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:56:37 smithi134 bash[61128]: debug 2024-07-15T06:56:37.638+0000 7f861f731040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-07-15T06:56:38.803 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:38 smithi134 bash[24068]: cluster 2024-07-15T06:56:36.965077+0000 mgr.y (mgr.25016) 36 : cluster [DBG] pgmap v20: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:56:38.803 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:38 smithi134 bash[24068]: audit 2024-07-15T06:56:37.865191+0000 mon.a (mon.0) 1107 : audit [DBG] from='client.? 172.21.15.89:0/3014720697' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:56:38.803 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:56:38 smithi134 bash[61128]: debug 2024-07-15T06:56:38.554+0000 7f861f731040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-07-15T06:56:38.803 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:56:38 smithi134 bash[61128]: debug 2024-07-15T06:56:38.638+0000 7f861f731040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-07-15T06:56:38.804 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:56:38 smithi134 bash[60729]: ts=2024-07-15T06:56:38.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1102-g9a202024 (9a2020246f85463d4fdb115c94aa40b463caa1d2) quincy (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1102-g9a202024 (9a2020246f85463d4fdb115c94aa40b463caa1d2) quincy (stable)\", cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:56:38.869 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:38 smithi089 bash[21793]: cluster 2024-07-15T06:56:36.965077+0000 mgr.y (mgr.25016) 36 : cluster [DBG] pgmap v20: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:56:38.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:38 smithi089 bash[21793]: audit 2024-07-15T06:56:37.865191+0000 mon.a (mon.0) 1107 : audit [DBG] from='client.? 172.21.15.89:0/3014720697' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:56:38.870 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:38 smithi089 bash[31372]: cluster 2024-07-15T06:56:36.965077+0000 mgr.y (mgr.25016) 36 : cluster [DBG] pgmap v20: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:56:38.870 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:38 smithi089 bash[31372]: audit 2024-07-15T06:56:37.865191+0000 mon.a (mon.0) 1107 : audit [DBG] from='client.? 172.21.15.89:0/3014720697' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:56:39.303 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:56:38 smithi134 bash[61128]: debug 2024-07-15T06:56:38.890+0000 7f861f731040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-07-15T06:56:39.303 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:56:38 smithi134 bash[61128]: debug 2024-07-15T06:56:38.966+0000 7f861f731040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-07-15T06:56:39.803 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:56:39 smithi134 bash[61128]: debug 2024-07-15T06:56:39.374+0000 7f861f731040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-07-15T06:56:39.803 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:56:39 smithi134 bash[61128]: debug 2024-07-15T06:56:39.454+0000 7f861f731040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-07-15T06:56:39.803 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:56:39 smithi134 bash[61128]: [15/Jul/2024:06:56:39] ENGINE Bus STARTING 2024-07-15T06:56:39.804 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:56:39 smithi134 bash[61128]: CherryPy Checker: 2024-07-15T06:56:39.804 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:56:39 smithi134 bash[61128]: The Application mounted at '' has an empty config. 2024-07-15T06:56:39.804 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:56:39 smithi134 bash[61128]: [15/Jul/2024:06:56:39] ENGINE Serving on http://:::9283 2024-07-15T06:56:39.804 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:56:39 smithi134 bash[61128]: [15/Jul/2024:06:56:39] ENGINE Bus STARTED 2024-07-15T06:56:40.303 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:39 smithi134 bash[24068]: audit 2024-07-15T06:56:38.911241+0000 mon.a (mon.0) 1108 : audit [INF] from='mgr.25016 ' entity='mgr.y' 2024-07-15T06:56:40.303 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:39 smithi134 bash[24068]: audit 2024-07-15T06:56:38.924493+0000 mon.a (mon.0) 1109 : audit [INF] from='mgr.25016 ' entity='mgr.y' 2024-07-15T06:56:40.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:39 smithi134 bash[24068]: cluster 2024-07-15T06:56:38.965795+0000 mgr.y (mgr.25016) 37 : cluster [DBG] pgmap v21: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:56:40.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:39 smithi134 bash[24068]: cluster 2024-07-15T06:56:39.482851+0000 mon.a (mon.0) 1110 : cluster [DBG] Standby manager daemon x restarted 2024-07-15T06:56:40.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:39 smithi134 bash[24068]: cluster 2024-07-15T06:56:39.483046+0000 mon.a (mon.0) 1111 : cluster [DBG] Standby manager daemon x started 2024-07-15T06:56:40.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:39 smithi134 bash[24068]: audit 2024-07-15T06:56:39.487812+0000 mon.b (mon.1) 118 : audit [DBG] from='mgr.? 172.21.15.134:0/1551694365' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-07-15T06:56:40.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:39 smithi134 bash[24068]: audit 2024-07-15T06:56:39.488681+0000 mon.b (mon.1) 119 : audit [DBG] from='mgr.? 172.21.15.134:0/1551694365' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-07-15T06:56:40.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:39 smithi134 bash[24068]: audit 2024-07-15T06:56:39.491015+0000 mon.b (mon.1) 120 : audit [DBG] from='mgr.? 172.21.15.134:0/1551694365' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-07-15T06:56:40.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:39 smithi134 bash[24068]: audit 2024-07-15T06:56:39.491746+0000 mon.b (mon.1) 121 : audit [DBG] from='mgr.? 172.21.15.134:0/1551694365' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-07-15T06:56:40.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:39 smithi134 bash[24068]: audit 2024-07-15T06:56:39.683844+0000 mon.a (mon.0) 1112 : audit [INF] from='mgr.25016 ' entity='mgr.y' 2024-07-15T06:56:40.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:39 smithi134 bash[24068]: audit 2024-07-15T06:56:39.697075+0000 mon.a (mon.0) 1113 : audit [INF] from='mgr.25016 ' entity='mgr.y' 2024-07-15T06:56:40.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:39 smithi089 bash[21793]: audit 2024-07-15T06:56:38.911241+0000 mon.a (mon.0) 1108 : audit [INF] from='mgr.25016 ' entity='mgr.y' 2024-07-15T06:56:40.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:39 smithi089 bash[21793]: audit 2024-07-15T06:56:38.924493+0000 mon.a (mon.0) 1109 : audit [INF] from='mgr.25016 ' entity='mgr.y' 2024-07-15T06:56:40.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:39 smithi089 bash[21793]: cluster 2024-07-15T06:56:38.965795+0000 mgr.y (mgr.25016) 37 : cluster [DBG] pgmap v21: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:56:40.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:39 smithi089 bash[21793]: cluster 2024-07-15T06:56:39.482851+0000 mon.a (mon.0) 1110 : cluster [DBG] Standby manager daemon x restarted 2024-07-15T06:56:40.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:39 smithi089 bash[21793]: cluster 2024-07-15T06:56:39.483046+0000 mon.a (mon.0) 1111 : cluster [DBG] Standby manager daemon x started 2024-07-15T06:56:40.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:39 smithi089 bash[21793]: audit 2024-07-15T06:56:39.487812+0000 mon.b (mon.1) 118 : audit [DBG] from='mgr.? 172.21.15.134:0/1551694365' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-07-15T06:56:40.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:39 smithi089 bash[21793]: audit 2024-07-15T06:56:39.488681+0000 mon.b (mon.1) 119 : audit [DBG] from='mgr.? 172.21.15.134:0/1551694365' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-07-15T06:56:40.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:39 smithi089 bash[21793]: audit 2024-07-15T06:56:39.491015+0000 mon.b (mon.1) 120 : audit [DBG] from='mgr.? 172.21.15.134:0/1551694365' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-07-15T06:56:40.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:39 smithi089 bash[21793]: audit 2024-07-15T06:56:39.491746+0000 mon.b (mon.1) 121 : audit [DBG] from='mgr.? 172.21.15.134:0/1551694365' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-07-15T06:56:40.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:39 smithi089 bash[21793]: audit 2024-07-15T06:56:39.683844+0000 mon.a (mon.0) 1112 : audit [INF] from='mgr.25016 ' entity='mgr.y' 2024-07-15T06:56:40.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:39 smithi089 bash[21793]: audit 2024-07-15T06:56:39.697075+0000 mon.a (mon.0) 1113 : audit [INF] from='mgr.25016 ' entity='mgr.y' 2024-07-15T06:56:40.371 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:39 smithi089 bash[31372]: audit 2024-07-15T06:56:38.911241+0000 mon.a (mon.0) 1108 : audit [INF] from='mgr.25016 ' entity='mgr.y' 2024-07-15T06:56:40.371 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:39 smithi089 bash[31372]: audit 2024-07-15T06:56:38.924493+0000 mon.a (mon.0) 1109 : audit [INF] from='mgr.25016 ' entity='mgr.y' 2024-07-15T06:56:40.371 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:39 smithi089 bash[31372]: cluster 2024-07-15T06:56:38.965795+0000 mgr.y (mgr.25016) 37 : cluster [DBG] pgmap v21: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:56:40.371 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:39 smithi089 bash[31372]: cluster 2024-07-15T06:56:39.482851+0000 mon.a (mon.0) 1110 : cluster [DBG] Standby manager daemon x restarted 2024-07-15T06:56:40.371 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:39 smithi089 bash[31372]: cluster 2024-07-15T06:56:39.483046+0000 mon.a (mon.0) 1111 : cluster [DBG] Standby manager daemon x started 2024-07-15T06:56:40.372 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:39 smithi089 bash[31372]: audit 2024-07-15T06:56:39.487812+0000 mon.b (mon.1) 118 : audit [DBG] from='mgr.? 172.21.15.134:0/1551694365' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-07-15T06:56:40.372 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:39 smithi089 bash[31372]: audit 2024-07-15T06:56:39.488681+0000 mon.b (mon.1) 119 : audit [DBG] from='mgr.? 172.21.15.134:0/1551694365' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-07-15T06:56:40.372 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:39 smithi089 bash[31372]: audit 2024-07-15T06:56:39.491015+0000 mon.b (mon.1) 120 : audit [DBG] from='mgr.? 172.21.15.134:0/1551694365' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-07-15T06:56:40.372 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:39 smithi089 bash[31372]: audit 2024-07-15T06:56:39.491746+0000 mon.b (mon.1) 121 : audit [DBG] from='mgr.? 172.21.15.134:0/1551694365' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-07-15T06:56:40.372 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:39 smithi089 bash[31372]: audit 2024-07-15T06:56:39.683844+0000 mon.a (mon.0) 1112 : audit [INF] from='mgr.25016 ' entity='mgr.y' 2024-07-15T06:56:40.372 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:39 smithi089 bash[31372]: audit 2024-07-15T06:56:39.697075+0000 mon.a (mon.0) 1113 : audit [INF] from='mgr.25016 ' entity='mgr.y' 2024-07-15T06:56:41.303 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:40 smithi134 bash[24068]: cluster 2024-07-15T06:56:39.934486+0000 mon.a (mon.0) 1114 : cluster [DBG] mgrmap e34: y(active, since 37s), standbys: x 2024-07-15T06:56:41.303 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:40 smithi134 bash[24068]: audit 2024-07-15T06:56:40.246779+0000 mon.c (mon.2) 65 : audit [DBG] from='client.? 172.21.15.89:0/3537856023' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:56:41.369 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:40 smithi089 bash[21793]: cluster 2024-07-15T06:56:39.934486+0000 mon.a (mon.0) 1114 : cluster [DBG] mgrmap e34: y(active, since 37s), standbys: x 2024-07-15T06:56:41.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:40 smithi089 bash[21793]: audit 2024-07-15T06:56:40.246779+0000 mon.c (mon.2) 65 : audit [DBG] from='client.? 172.21.15.89:0/3537856023' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:56:41.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:40 smithi089 bash[31372]: cluster 2024-07-15T06:56:39.934486+0000 mon.a (mon.0) 1114 : cluster [DBG] mgrmap e34: y(active, since 37s), standbys: x 2024-07-15T06:56:41.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:40 smithi089 bash[31372]: audit 2024-07-15T06:56:40.246779+0000 mon.c (mon.2) 65 : audit [DBG] from='client.? 172.21.15.89:0/3537856023' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:56:42.303 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:41 smithi134 bash[24068]: cluster 2024-07-15T06:56:40.966893+0000 mgr.y (mgr.25016) 38 : cluster [DBG] pgmap v22: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:56:42.369 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:41 smithi089 bash[21793]: cluster 2024-07-15T06:56:40.966893+0000 mgr.y (mgr.25016) 38 : cluster [DBG] pgmap v22: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:56:42.369 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:41 smithi089 bash[31372]: cluster 2024-07-15T06:56:40.966893+0000 mgr.y (mgr.25016) 38 : cluster [DBG] pgmap v22: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:56:42.803 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:56:42 smithi134 bash[60729]: ts=2024-07-15T06:56:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:56:43.303 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:42 smithi134 bash[24068]: audit 2024-07-15T06:56:42.620043+0000 mon.a (mon.0) 1115 : audit [DBG] from='client.? 172.21.15.89:0/220453168' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:56:43.369 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:42 smithi089 bash[21793]: audit 2024-07-15T06:56:42.620043+0000 mon.a (mon.0) 1115 : audit [DBG] from='client.? 172.21.15.89:0/220453168' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:56:43.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:42 smithi089 bash[31372]: audit 2024-07-15T06:56:42.620043+0000 mon.a (mon.0) 1115 : audit [DBG] from='client.? 172.21.15.89:0/220453168' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:56:43.803 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:56:43 smithi134 bash[60729]: ts=2024-07-15T06:56:43.508Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi134\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi134\", job=\"node\", machine=\"x86_64\", nodename=\"smithi134\", release=\"5.15.0-113-generic\", sysname=\"Linux\", version=\"#123-Ubuntu SMP Mon Jun 10 08:16:17 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", domainname=\"(none)\", instance=\"smithi134\", job=\"node\", machine=\"x86_64\", nodename=\"smithi134\", release=\"5.15.0-113-generic\", sysname=\"Linux\", version=\"#123-Ubuntu SMP Mon Jun 10 08:16:17 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:56:44.303 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:43 smithi134 bash[24068]: cluster 2024-07-15T06:56:42.967593+0000 mgr.y (mgr.25016) 39 : cluster [DBG] pgmap v23: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:56:44.369 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:43 smithi089 bash[21793]: cluster 2024-07-15T06:56:42.967593+0000 mgr.y (mgr.25016) 39 : cluster [DBG] pgmap v23: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:56:44.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:43 smithi089 bash[31372]: cluster 2024-07-15T06:56:42.967593+0000 mgr.y (mgr.25016) 39 : cluster [DBG] pgmap v23: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:56:44.370 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:56:44 smithi089 bash[72894]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:56:44] "GET /metrics HTTP/1.1" 200 33797 "" "Prometheus/2.43.0" 2024-07-15T06:56:44.803 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:56:44 smithi134 bash[60729]: ts=2024-07-15T06:56:44.453Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:56:44.803 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:56:44 smithi134 bash[60729]: ts=2024-07-15T06:56:44.454Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:56:45.303 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:45 smithi134 bash[24068]: audit 2024-07-15T06:56:44.991001+0000 mon.c (mon.2) 66 : audit [DBG] from='client.? 172.21.15.89:0/2428860278' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:56:45.369 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:45 smithi089 bash[21793]: audit 2024-07-15T06:56:44.991001+0000 mon.c (mon.2) 66 : audit [DBG] from='client.? 172.21.15.89:0/2428860278' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:56:45.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:45 smithi089 bash[31372]: audit 2024-07-15T06:56:44.991001+0000 mon.c (mon.2) 66 : audit [DBG] from='client.? 172.21.15.89:0/2428860278' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:56:46.303 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:46 smithi134 bash[24068]: cluster 2024-07-15T06:56:44.968663+0000 mgr.y (mgr.25016) 40 : cluster [DBG] pgmap v24: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:56:46.369 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:46 smithi089 bash[21793]: cluster 2024-07-15T06:56:44.968663+0000 mgr.y (mgr.25016) 40 : cluster [DBG] pgmap v24: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:56:46.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:46 smithi089 bash[31372]: cluster 2024-07-15T06:56:44.968663+0000 mgr.y (mgr.25016) 40 : cluster [DBG] pgmap v24: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:56:47.719 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:47 smithi134 bash[24068]: audit 2024-07-15T06:56:47.354032+0000 mon.c (mon.2) 67 : audit [DBG] from='client.? 172.21.15.89:0/4262340850' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:56:47.869 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:47 smithi089 bash[21793]: audit 2024-07-15T06:56:47.354032+0000 mon.c (mon.2) 67 : audit [DBG] from='client.? 172.21.15.89:0/4262340850' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:56:47.870 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:47 smithi089 bash[31372]: audit 2024-07-15T06:56:47.354032+0000 mon.c (mon.2) 67 : audit [DBG] from='client.? 172.21.15.89:0/4262340850' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:56:48.803 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:48 smithi134 bash[24068]: cluster 2024-07-15T06:56:46.969318+0000 mgr.y (mgr.25016) 41 : cluster [DBG] pgmap v25: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:56:48.803 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:48 smithi134 bash[24068]: audit 2024-07-15T06:56:48.173614+0000 mon.b (mon.1) 122 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:56:48.804 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:56:48 smithi134 bash[60729]: ts=2024-07-15T06:56:48.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1102-g9a202024 (9a2020246f85463d4fdb115c94aa40b463caa1d2) quincy (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1102-g9a202024 (9a2020246f85463d4fdb115c94aa40b463caa1d2) quincy (stable)\", cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:56:48.869 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:48 smithi089 bash[21793]: cluster 2024-07-15T06:56:46.969318+0000 mgr.y (mgr.25016) 41 : cluster [DBG] pgmap v25: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:56:48.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:48 smithi089 bash[21793]: audit 2024-07-15T06:56:48.173614+0000 mon.b (mon.1) 122 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:56:48.870 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:48 smithi089 bash[31372]: cluster 2024-07-15T06:56:46.969318+0000 mgr.y (mgr.25016) 41 : cluster [DBG] pgmap v25: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:56:48.870 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:48 smithi089 bash[31372]: audit 2024-07-15T06:56:48.173614+0000 mon.b (mon.1) 122 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:56:50.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:50 smithi134 bash[24068]: cluster 2024-07-15T06:56:48.970033+0000 mgr.y (mgr.25016) 42 : cluster [DBG] pgmap v26: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:56:50.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:50 smithi134 bash[24068]: audit 2024-07-15T06:56:49.379746+0000 mon.a (mon.0) 1116 : audit [INF] from='mgr.25016 ' entity='mgr.y' 2024-07-15T06:56:50.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:50 smithi134 bash[24068]: audit 2024-07-15T06:56:49.391301+0000 mon.a (mon.0) 1117 : audit [INF] from='mgr.25016 ' entity='mgr.y' 2024-07-15T06:56:50.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:50 smithi134 bash[24068]: audit 2024-07-15T06:56:49.396701+0000 mon.b (mon.1) 123 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:56:50.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:50 smithi134 bash[24068]: audit 2024-07-15T06:56:49.398209+0000 mon.b (mon.1) 124 : audit [INF] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:56:50.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:50 smithi134 bash[24068]: audit 2024-07-15T06:56:49.405304+0000 mon.a (mon.0) 1118 : audit [INF] from='mgr.25016 ' entity='mgr.y' 2024-07-15T06:56:50.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:50 smithi134 bash[24068]: audit 2024-07-15T06:56:49.553604+0000 mon.b (mon.1) 125 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:56:50.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:50 smithi134 bash[24068]: audit 2024-07-15T06:56:49.560247+0000 mon.b (mon.1) 126 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-15T06:56:50.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:50 smithi134 bash[24068]: cephadm 2024-07-15T06:56:49.560282+0000 mgr.y (mgr.25016) 43 : cephadm [INF] Upgrade: Setting container_image for all mgr 2024-07-15T06:56:50.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:50 smithi134 bash[24068]: audit 2024-07-15T06:56:49.569454+0000 mon.a (mon.0) 1119 : audit [INF] from='mgr.25016 ' entity='mgr.y' 2024-07-15T06:56:50.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:50 smithi134 bash[24068]: audit 2024-07-15T06:56:49.572794+0000 mon.a (mon.0) 1120 : audit [INF] from='mgr.25016 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-07-15T06:56:50.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:50 smithi134 bash[24068]: audit 2024-07-15T06:56:49.573609+0000 mon.b (mon.1) 127 : audit [INF] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-07-15T06:56:50.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:50 smithi134 bash[24068]: audit 2024-07-15T06:56:49.580165+0000 mon.a (mon.0) 1121 : audit [INF] from='mgr.25016 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-07-15T06:56:50.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:50 smithi134 bash[24068]: audit 2024-07-15T06:56:49.583691+0000 mon.a (mon.0) 1122 : audit [INF] from='mgr.25016 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-07-15T06:56:50.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:50 smithi134 bash[24068]: audit 2024-07-15T06:56:49.584644+0000 mon.b (mon.1) 128 : audit [INF] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-07-15T06:56:50.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:50 smithi134 bash[24068]: audit 2024-07-15T06:56:49.591005+0000 mon.a (mon.0) 1123 : audit [INF] from='mgr.25016 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-07-15T06:56:50.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:50 smithi134 bash[24068]: audit 2024-07-15T06:56:49.596380+0000 mon.b (mon.1) 129 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-07-15T06:56:50.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:50 smithi134 bash[24068]: audit 2024-07-15T06:56:49.598510+0000 mon.b (mon.1) 130 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-07-15T06:56:50.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:50 smithi134 bash[24068]: cephadm 2024-07-15T06:56:49.598599+0000 mgr.y (mgr.25016) 44 : cephadm [INF] Upgrade: It appears safe to stop mon.a 2024-07-15T06:56:50.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:50 smithi134 bash[24068]: audit 2024-07-15T06:56:49.763600+0000 mon.c (mon.2) 68 : audit [DBG] from='client.? 172.21.15.89:0/2135791043' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:56:50.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:50 smithi089 bash[21793]: cluster 2024-07-15T06:56:48.970033+0000 mgr.y (mgr.25016) 42 : cluster [DBG] pgmap v26: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:56:50.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:50 smithi089 bash[21793]: audit 2024-07-15T06:56:49.379746+0000 mon.a (mon.0) 1116 : audit [INF] from='mgr.25016 ' entity='mgr.y' 2024-07-15T06:56:50.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:50 smithi089 bash[21793]: audit 2024-07-15T06:56:49.391301+0000 mon.a (mon.0) 1117 : audit [INF] from='mgr.25016 ' entity='mgr.y' 2024-07-15T06:56:50.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:50 smithi089 bash[21793]: audit 2024-07-15T06:56:49.396701+0000 mon.b (mon.1) 123 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:56:50.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:50 smithi089 bash[21793]: audit 2024-07-15T06:56:49.398209+0000 mon.b (mon.1) 124 : audit [INF] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:56:50.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:50 smithi089 bash[21793]: audit 2024-07-15T06:56:49.405304+0000 mon.a (mon.0) 1118 : audit [INF] from='mgr.25016 ' entity='mgr.y' 2024-07-15T06:56:50.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:50 smithi089 bash[21793]: audit 2024-07-15T06:56:49.553604+0000 mon.b (mon.1) 125 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:56:50.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:50 smithi089 bash[21793]: audit 2024-07-15T06:56:49.560247+0000 mon.b (mon.1) 126 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-15T06:56:50.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:50 smithi089 bash[21793]: cephadm 2024-07-15T06:56:49.560282+0000 mgr.y (mgr.25016) 43 : cephadm [INF] Upgrade: Setting container_image for all mgr 2024-07-15T06:56:50.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:50 smithi089 bash[21793]: audit 2024-07-15T06:56:49.569454+0000 mon.a (mon.0) 1119 : audit [INF] from='mgr.25016 ' entity='mgr.y' 2024-07-15T06:56:50.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:50 smithi089 bash[21793]: audit 2024-07-15T06:56:49.572794+0000 mon.a (mon.0) 1120 : audit [INF] from='mgr.25016 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-07-15T06:56:50.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:50 smithi089 bash[21793]: audit 2024-07-15T06:56:49.573609+0000 mon.b (mon.1) 127 : audit [INF] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-07-15T06:56:50.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:50 smithi089 bash[21793]: audit 2024-07-15T06:56:49.580165+0000 mon.a (mon.0) 1121 : audit [INF] from='mgr.25016 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-07-15T06:56:50.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:50 smithi089 bash[21793]: audit 2024-07-15T06:56:49.583691+0000 mon.a (mon.0) 1122 : audit [INF] from='mgr.25016 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-07-15T06:56:50.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:50 smithi089 bash[21793]: audit 2024-07-15T06:56:49.584644+0000 mon.b (mon.1) 128 : audit [INF] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-07-15T06:56:50.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:50 smithi089 bash[21793]: audit 2024-07-15T06:56:49.591005+0000 mon.a (mon.0) 1123 : audit [INF] from='mgr.25016 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-07-15T06:56:50.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:50 smithi089 bash[21793]: audit 2024-07-15T06:56:49.596380+0000 mon.b (mon.1) 129 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-07-15T06:56:50.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:50 smithi089 bash[21793]: audit 2024-07-15T06:56:49.598510+0000 mon.b (mon.1) 130 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-07-15T06:56:50.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:50 smithi089 bash[21793]: cephadm 2024-07-15T06:56:49.598599+0000 mgr.y (mgr.25016) 44 : cephadm [INF] Upgrade: It appears safe to stop mon.a 2024-07-15T06:56:50.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:50 smithi089 bash[21793]: audit 2024-07-15T06:56:49.763600+0000 mon.c (mon.2) 68 : audit [DBG] from='client.? 172.21.15.89:0/2135791043' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:56:50.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:50 smithi089 bash[31372]: cluster 2024-07-15T06:56:48.970033+0000 mgr.y (mgr.25016) 42 : cluster [DBG] pgmap v26: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:56:50.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:50 smithi089 bash[31372]: audit 2024-07-15T06:56:49.379746+0000 mon.a (mon.0) 1116 : audit [INF] from='mgr.25016 ' entity='mgr.y' 2024-07-15T06:56:50.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:50 smithi089 bash[31372]: audit 2024-07-15T06:56:49.391301+0000 mon.a (mon.0) 1117 : audit [INF] from='mgr.25016 ' entity='mgr.y' 2024-07-15T06:56:50.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:50 smithi089 bash[31372]: audit 2024-07-15T06:56:49.396701+0000 mon.b (mon.1) 123 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:56:50.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:50 smithi089 bash[31372]: audit 2024-07-15T06:56:49.398209+0000 mon.b (mon.1) 124 : audit [INF] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:56:50.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:50 smithi089 bash[31372]: audit 2024-07-15T06:56:49.405304+0000 mon.a (mon.0) 1118 : audit [INF] from='mgr.25016 ' entity='mgr.y' 2024-07-15T06:56:50.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:50 smithi089 bash[31372]: audit 2024-07-15T06:56:49.553604+0000 mon.b (mon.1) 125 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:56:50.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:50 smithi089 bash[31372]: audit 2024-07-15T06:56:49.560247+0000 mon.b (mon.1) 126 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-15T06:56:50.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:50 smithi089 bash[31372]: cephadm 2024-07-15T06:56:49.560282+0000 mgr.y (mgr.25016) 43 : cephadm [INF] Upgrade: Setting container_image for all mgr 2024-07-15T06:56:50.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:50 smithi089 bash[31372]: audit 2024-07-15T06:56:49.569454+0000 mon.a (mon.0) 1119 : audit [INF] from='mgr.25016 ' entity='mgr.y' 2024-07-15T06:56:50.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:50 smithi089 bash[31372]: audit 2024-07-15T06:56:49.572794+0000 mon.a (mon.0) 1120 : audit [INF] from='mgr.25016 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-07-15T06:56:50.873 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:50 smithi089 bash[31372]: audit 2024-07-15T06:56:49.573609+0000 mon.b (mon.1) 127 : audit [INF] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-07-15T06:56:50.873 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:50 smithi089 bash[31372]: audit 2024-07-15T06:56:49.580165+0000 mon.a (mon.0) 1121 : audit [INF] from='mgr.25016 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-07-15T06:56:50.873 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:50 smithi089 bash[31372]: audit 2024-07-15T06:56:49.583691+0000 mon.a (mon.0) 1122 : audit [INF] from='mgr.25016 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-07-15T06:56:50.873 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:50 smithi089 bash[31372]: audit 2024-07-15T06:56:49.584644+0000 mon.b (mon.1) 128 : audit [INF] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-07-15T06:56:50.873 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:50 smithi089 bash[31372]: audit 2024-07-15T06:56:49.591005+0000 mon.a (mon.0) 1123 : audit [INF] from='mgr.25016 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-07-15T06:56:50.873 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:50 smithi089 bash[31372]: audit 2024-07-15T06:56:49.596380+0000 mon.b (mon.1) 129 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-07-15T06:56:50.873 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:50 smithi089 bash[31372]: audit 2024-07-15T06:56:49.598510+0000 mon.b (mon.1) 130 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-07-15T06:56:50.873 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:50 smithi089 bash[31372]: cephadm 2024-07-15T06:56:49.598599+0000 mgr.y (mgr.25016) 44 : cephadm [INF] Upgrade: It appears safe to stop mon.a 2024-07-15T06:56:50.873 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:50 smithi089 bash[31372]: audit 2024-07-15T06:56:49.763600+0000 mon.c (mon.2) 68 : audit [DBG] from='client.? 172.21.15.89:0/2135791043' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:56:52.803 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:56:52 smithi134 bash[60729]: ts=2024-07-15T06:56:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:56:52.803 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:52 smithi134 bash[24068]: cluster 2024-07-15T06:56:50.971085+0000 mgr.y (mgr.25016) 45 : cluster [DBG] pgmap v27: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:56:52.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:52 smithi134 bash[24068]: audit 2024-07-15T06:56:52.022544+0000 mon.a (mon.0) 1124 : audit [INF] from='mgr.25016 ' entity='mgr.y' 2024-07-15T06:56:52.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:52 smithi134 bash[24068]: audit 2024-07-15T06:56:52.029745+0000 mon.b (mon.1) 131 : audit [INF] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-07-15T06:56:52.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:52 smithi134 bash[24068]: audit 2024-07-15T06:56:52.031191+0000 mon.b (mon.1) 132 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-07-15T06:56:52.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:52 smithi134 bash[24068]: audit 2024-07-15T06:56:52.032572+0000 mon.b (mon.1) 133 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:56:52.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:52 smithi134 bash[24068]: audit 2024-07-15T06:56:52.140672+0000 mon.a (mon.0) 1125 : audit [DBG] from='client.? 172.21.15.89:0/2211116996' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:56:52.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:52 smithi089 bash[21793]: cluster 2024-07-15T06:56:50.971085+0000 mgr.y (mgr.25016) 45 : cluster [DBG] pgmap v27: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:56:52.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:52 smithi089 bash[21793]: audit 2024-07-15T06:56:52.022544+0000 mon.a (mon.0) 1124 : audit [INF] from='mgr.25016 ' entity='mgr.y' 2024-07-15T06:56:52.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:52 smithi089 bash[21793]: audit 2024-07-15T06:56:52.029745+0000 mon.b (mon.1) 131 : audit [INF] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-07-15T06:56:52.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:52 smithi089 bash[21793]: audit 2024-07-15T06:56:52.031191+0000 mon.b (mon.1) 132 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-07-15T06:56:52.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:52 smithi089 bash[21793]: audit 2024-07-15T06:56:52.032572+0000 mon.b (mon.1) 133 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:56:52.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:52 smithi089 bash[21793]: audit 2024-07-15T06:56:52.140672+0000 mon.a (mon.0) 1125 : audit [DBG] from='client.? 172.21.15.89:0/2211116996' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:56:52.870 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:52 smithi089 bash[31372]: cluster 2024-07-15T06:56:50.971085+0000 mgr.y (mgr.25016) 45 : cluster [DBG] pgmap v27: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:56:52.870 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:52 smithi089 bash[31372]: audit 2024-07-15T06:56:52.022544+0000 mon.a (mon.0) 1124 : audit [INF] from='mgr.25016 ' entity='mgr.y' 2024-07-15T06:56:52.870 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:52 smithi089 bash[31372]: audit 2024-07-15T06:56:52.029745+0000 mon.b (mon.1) 131 : audit [INF] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-07-15T06:56:52.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:52 smithi089 bash[31372]: audit 2024-07-15T06:56:52.031191+0000 mon.b (mon.1) 132 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-07-15T06:56:52.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:52 smithi089 bash[31372]: audit 2024-07-15T06:56:52.032572+0000 mon.b (mon.1) 133 : audit [DBG] from='mgr.25016 172.21.15.89:0/3485196172' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:56:52.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:52 smithi089 bash[31372]: audit 2024-07-15T06:56:52.140672+0000 mon.a (mon.0) 1125 : audit [DBG] from='client.? 172.21.15.89:0/2211116996' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:56:53.638 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:53 smithi089 bash[21793]: cephadm 2024-07-15T06:56:52.016613+0000 mgr.y (mgr.25016) 46 : cephadm [INF] Upgrade: Updating mon.a 2024-07-15T06:56:53.639 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:53 smithi089 bash[21793]: cephadm 2024-07-15T06:56:52.032571+0000 mgr.y (mgr.25016) 47 : cephadm [INF] Deploying daemon mon.a on smithi089 2024-07-15T06:56:53.639 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:53 smithi089 bash[31372]: cephadm 2024-07-15T06:56:52.016613+0000 mgr.y (mgr.25016) 46 : cephadm [INF] Upgrade: Updating mon.a 2024-07-15T06:56:53.639 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:53 smithi089 bash[31372]: cephadm 2024-07-15T06:56:52.032571+0000 mgr.y (mgr.25016) 47 : cephadm [INF] Deploying daemon mon.a on smithi089 2024-07-15T06:56:53.803 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:53 smithi134 bash[24068]: cephadm 2024-07-15T06:56:52.016613+0000 mgr.y (mgr.25016) 46 : cephadm [INF] Upgrade: Updating mon.a 2024-07-15T06:56:53.803 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:53 smithi134 bash[24068]: cephadm 2024-07-15T06:56:52.032571+0000 mgr.y (mgr.25016) 47 : cephadm [INF] Deploying daemon mon.a on smithi089 2024-07-15T06:56:53.803 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:56:53 smithi134 bash[60729]: ts=2024-07-15T06:56:53.507Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi134\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi134\", job=\"node\", machine=\"x86_64\", nodename=\"smithi134\", release=\"5.15.0-113-generic\", sysname=\"Linux\", version=\"#123-Ubuntu SMP Mon Jun 10 08:16:17 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", domainname=\"(none)\", instance=\"smithi134\", job=\"node\", machine=\"x86_64\", nodename=\"smithi134\", release=\"5.15.0-113-generic\", sysname=\"Linux\", version=\"#123-Ubuntu SMP Mon Jun 10 08:16:17 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:56:54.528 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:54 smithi089 bash[21793]: cluster 2024-07-15T06:56:52.971737+0000 mgr.y (mgr.25016) 48 : cluster [DBG] pgmap v28: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:56:54.528 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:54 smithi089 bash[31372]: cluster 2024-07-15T06:56:52.971737+0000 mgr.y (mgr.25016) 48 : cluster [DBG] pgmap v28: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:56:54.529 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:56:54 smithi089 bash[72894]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:56:54] "GET /metrics HTTP/1.1" 200 33793 "" "Prometheus/2.43.0" 2024-07-15T06:56:54.803 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:54 smithi134 bash[24068]: cluster 2024-07-15T06:56:52.971737+0000 mgr.y (mgr.25016) 48 : cluster [DBG] pgmap v28: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:56:54.803 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:56:54 smithi134 bash[60729]: ts=2024-07-15T06:56:54.454Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:56:54.803 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:56:54 smithi134 bash[60729]: ts=2024-07-15T06:56:54.454Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:56:54.870 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:56:54 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:56:54.901 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:54 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:56:54.902 INFO:journalctl@ceph.osd.0.smithi089.stdout:Jul 15 06:56:54 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:56:54.902 INFO:journalctl@ceph.osd.2.smithi089.stdout:Jul 15 06:56:54 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:56:54.903 INFO:journalctl@ceph.osd.3.smithi089.stdout:Jul 15 06:56:54 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:56:54.903 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:56:54 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:56:54.903 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 06:56:54 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:56:54.903 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:54 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:56:54.904 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:54 smithi089 systemd[1]: Stopping Ceph mon.a for 75ba02bc-4274-11ef-bcad-c7b262605968... 2024-07-15T06:56:54.904 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:54 smithi089 bash[21793]: debug 2024-07-15T06:56:54.755+0000 7fb0c1620640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2024-07-15T06:56:54.904 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:54 smithi089 bash[21793]: debug 2024-07-15T06:56:54.755+0000 7fb0c1620640 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2024-07-15T06:56:54.904 INFO:journalctl@ceph.osd.1.smithi089.stdout:Jul 15 06:56:54 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:56:55.803 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:55 smithi089 bash[79693]: ceph-75ba02bc-4274-11ef-bcad-c7b262605968-mon-a 2024-07-15T06:56:55.803 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:55 smithi089 bash[79753]: Error response from daemon: No such container: ceph-75ba02bc-4274-11ef-bcad-c7b262605968-mon-a 2024-07-15T06:56:55.803 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:55 smithi089 systemd[1]: ceph-75ba02bc-4274-11ef-bcad-c7b262605968@mon.a.service: Deactivated successfully. 2024-07-15T06:56:55.803 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:55 smithi089 systemd[1]: Stopped Ceph mon.a for 75ba02bc-4274-11ef-bcad-c7b262605968. 2024-07-15T06:56:56.119 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:56:55 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:56:56.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:55 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:56:56.120 INFO:journalctl@ceph.osd.0.smithi089.stdout:Jul 15 06:56:55 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:56:56.120 INFO:journalctl@ceph.osd.1.smithi089.stdout:Jul 15 06:56:55 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:56:56.121 INFO:journalctl@ceph.osd.2.smithi089.stdout:Jul 15 06:56:55 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:56:56.121 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:56:55 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:56:56.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:55 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:56:56.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:55 smithi089 systemd[1]: Started Ceph mon.a for 75ba02bc-4274-11ef-bcad-c7b262605968. 2024-07-15T06:56:56.122 INFO:journalctl@ceph.osd.3.smithi089.stdout:Jul 15 06:56:55 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:56:56.122 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 06:56:55 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:56:57.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.199+0000 7fb568434b00 0 set uid:gid to 167:167 (ceph:ceph) 2024-07-15T06:56:57.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.199+0000 7fb568434b00 0 ceph version 18.2.2-1609-g6ea7aaa9 (6ea7aaa9f93d34dc4d21862afe00140dfddbd706) reef (stable), process ceph-mon, pid 7 2024-07-15T06:56:57.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.199+0000 7fb568434b00 0 pidfile_write: ignore empty --pid-file 2024-07-15T06:56:57.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.199+0000 7fb568434b00 0 load: jerasure load: lrc 2024-07-15T06:56:57.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: RocksDB version: 7.9.2 2024-07-15T06:56:57.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Git sha 0 2024-07-15T06:56:57.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Compile date 2024-07-05 15:13:19 2024-07-15T06:56:57.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: DB SUMMARY 2024-07-15T06:56:57.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: DB Session ID: MLJ68VVVPYC33U9MPK23 2024-07-15T06:56:57.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: CURRENT file: CURRENT 2024-07-15T06:56:57.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: IDENTITY file: IDENTITY 2024-07-15T06:56:57.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: MANIFEST file: MANIFEST-000015 size: 1734 Bytes 2024-07-15T06:56:57.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000042.sst 2024-07-15T06:56:57.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000040.log size: 135533 ; 2024-07-15T06:56:57.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.error_if_exists: 0 2024-07-15T06:56:57.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.create_if_missing: 0 2024-07-15T06:56:57.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.paranoid_checks: 1 2024-07-15T06:56:57.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.flush_verify_memtable_count: 1 2024-07-15T06:56:57.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-07-15T06:56:57.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-07-15T06:56:57.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.env: 0x55a733c6eba0 2024-07-15T06:56:57.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.fs: PosixFileSystem 2024-07-15T06:56:57.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.info_log: 0x55a7357a0be0 2024-07-15T06:56:57.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.max_file_opening_threads: 16 2024-07-15T06:56:57.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.statistics: (nil) 2024-07-15T06:56:57.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.use_fsync: 0 2024-07-15T06:56:57.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.max_log_file_size: 0 2024-07-15T06:56:57.373 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.max_manifest_file_size: 1073741824 2024-07-15T06:56:57.373 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.log_file_time_to_roll: 0 2024-07-15T06:56:57.373 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.keep_log_file_num: 1000 2024-07-15T06:56:57.373 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.recycle_log_file_num: 0 2024-07-15T06:56:57.373 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.allow_fallocate: 1 2024-07-15T06:56:57.374 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.allow_mmap_reads: 0 2024-07-15T06:56:57.374 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.allow_mmap_writes: 0 2024-07-15T06:56:57.374 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.use_direct_reads: 0 2024-07-15T06:56:57.374 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-07-15T06:56:57.374 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.create_missing_column_families: 0 2024-07-15T06:56:57.374 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.db_log_dir: 2024-07-15T06:56:57.374 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.wal_dir: 2024-07-15T06:56:57.374 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.table_cache_numshardbits: 6 2024-07-15T06:56:57.374 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.WAL_ttl_seconds: 0 2024-07-15T06:56:57.374 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.WAL_size_limit_MB: 0 2024-07-15T06:56:57.374 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-07-15T06:56:57.375 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.manifest_preallocation_size: 4194304 2024-07-15T06:56:57.375 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.is_fd_close_on_exec: 1 2024-07-15T06:56:57.375 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.advise_random_on_open: 1 2024-07-15T06:56:57.375 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.db_write_buffer_size: 0 2024-07-15T06:56:57.375 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.write_buffer_manager: 0x55a7357aeb40 2024-07-15T06:56:57.375 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.access_hint_on_compaction_start: 1 2024-07-15T06:56:57.375 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2024-07-15T06:56:57.375 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.use_adaptive_mutex: 0 2024-07-15T06:56:57.375 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.rate_limiter: (nil) 2024-07-15T06:56:57.375 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-07-15T06:56:57.375 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.wal_recovery_mode: 2 2024-07-15T06:56:57.375 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.enable_thread_tracking: 0 2024-07-15T06:56:57.376 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.enable_pipelined_write: 0 2024-07-15T06:56:57.376 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.unordered_write: 0 2024-07-15T06:56:57.376 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2024-07-15T06:56:57.376 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-07-15T06:56:57.376 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.write_thread_max_yield_usec: 100 2024-07-15T06:56:57.376 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2024-07-15T06:56:57.376 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.row_cache: None 2024-07-15T06:56:57.376 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.wal_filter: None 2024-07-15T06:56:57.376 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.avoid_flush_during_recovery: 0 2024-07-15T06:56:57.376 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.allow_ingest_behind: 0 2024-07-15T06:56:57.376 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.two_write_queues: 0 2024-07-15T06:56:57.376 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.manual_wal_flush: 0 2024-07-15T06:56:57.376 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.wal_compression: 0 2024-07-15T06:56:57.377 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.atomic_flush: 0 2024-07-15T06:56:57.377 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-07-15T06:56:57.377 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.persist_stats_to_disk: 0 2024-07-15T06:56:57.377 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.write_dbid_to_manifest: 0 2024-07-15T06:56:57.377 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.log_readahead_size: 0 2024-07-15T06:56:57.377 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.file_checksum_gen_factory: Unknown 2024-07-15T06:56:57.377 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.best_efforts_recovery: 0 2024-07-15T06:56:57.377 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-07-15T06:56:57.377 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-07-15T06:56:57.377 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.allow_data_in_errors: 0 2024-07-15T06:56:57.377 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.db_host_id: __hostname__ 2024-07-15T06:56:57.377 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.enforce_single_del_contracts: true 2024-07-15T06:56:57.378 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.max_background_jobs: 2 2024-07-15T06:56:57.378 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.max_background_compactions: -1 2024-07-15T06:56:57.378 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.max_subcompactions: 1 2024-07-15T06:56:57.378 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2024-07-15T06:56:57.378 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-07-15T06:56:57.378 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.delayed_write_rate : 16777216 2024-07-15T06:56:57.378 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.max_total_wal_size: 0 2024-07-15T06:56:57.378 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-07-15T06:56:57.378 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.stats_dump_period_sec: 600 2024-07-15T06:56:57.378 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.stats_persist_period_sec: 600 2024-07-15T06:56:57.378 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.stats_history_buffer_size: 1048576 2024-07-15T06:56:57.378 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.max_open_files: -1 2024-07-15T06:56:57.379 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.bytes_per_sync: 0 2024-07-15T06:56:57.379 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.wal_bytes_per_sync: 0 2024-07-15T06:56:57.379 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.strict_bytes_per_sync: 0 2024-07-15T06:56:57.379 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.compaction_readahead_size: 0 2024-07-15T06:56:57.379 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.max_background_flushes: -1 2024-07-15T06:56:57.379 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Compression algorithms supported: 2024-07-15T06:56:57.379 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: kZSTD supported: 0 2024-07-15T06:56:57.379 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: kXpressCompression supported: 0 2024-07-15T06:56:57.379 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: kBZip2Compression supported: 0 2024-07-15T06:56:57.379 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: kZSTDNotFinalCompression supported: 0 2024-07-15T06:56:57.379 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: kLZ4Compression supported: 1 2024-07-15T06:56:57.379 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: kZlibCompression supported: 1 2024-07-15T06:56:57.379 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: kLZ4HCCompression supported: 1 2024-07-15T06:56:57.379 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: kSnappyCompression supported: 1 2024-07-15T06:56:57.379 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Fast CRC32 supported: Supported on x86 2024-07-15T06:56:57.380 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: DMutex implementation: pthread_mutex_t 2024-07-15T06:56:57.380 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 2024-07-15T06:56:57.380 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-07-15T06:56:57.380 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-07-15T06:56:57.380 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.merge_operator: 2024-07-15T06:56:57.380 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.compaction_filter: None 2024-07-15T06:56:57.380 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.compaction_filter_factory: None 2024-07-15T06:56:57.380 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.sst_partitioner_factory: None 2024-07-15T06:56:57.380 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.memtable_factory: SkipListFactory 2024-07-15T06:56:57.380 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.table_factory: BlockBasedTable 2024-07-15T06:56:57.380 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55a7357a0820) 2024-07-15T06:56:57.380 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: cache_index_and_filter_blocks: 1 2024-07-15T06:56:57.380 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: cache_index_and_filter_blocks_with_high_priority: 0 2024-07-15T06:56:57.381 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: pin_l0_filter_and_index_blocks_in_cache: 0 2024-07-15T06:56:57.381 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: pin_top_level_index_and_filter: 1 2024-07-15T06:56:57.381 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: index_type: 0 2024-07-15T06:56:57.381 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: data_block_index_type: 0 2024-07-15T06:56:57.381 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: index_shortening: 1 2024-07-15T06:56:57.381 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: data_block_hash_table_util_ratio: 0.750000 2024-07-15T06:56:57.381 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: checksum: 4 2024-07-15T06:56:57.381 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: no_block_cache: 0 2024-07-15T06:56:57.381 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: block_cache: 0x55a7357951f0 2024-07-15T06:56:57.381 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: block_cache_name: BinnedLRUCache 2024-07-15T06:56:57.381 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: block_cache_options: 2024-07-15T06:56:57.381 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: capacity : 536870912 2024-07-15T06:56:57.381 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: num_shard_bits : 4 2024-07-15T06:56:57.381 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: strict_capacity_limit : 0 2024-07-15T06:56:57.382 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: high_pri_pool_ratio: 0.000 2024-07-15T06:56:57.382 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: block_cache_compressed: (nil) 2024-07-15T06:56:57.382 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: persistent_cache: (nil) 2024-07-15T06:56:57.382 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: block_size: 4096 2024-07-15T06:56:57.382 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: block_size_deviation: 10 2024-07-15T06:56:57.382 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: block_restart_interval: 16 2024-07-15T06:56:57.382 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: index_block_restart_interval: 1 2024-07-15T06:56:57.382 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: metadata_block_size: 4096 2024-07-15T06:56:57.382 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: partition_filters: 0 2024-07-15T06:56:57.382 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: use_delta_encoding: 1 2024-07-15T06:56:57.382 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: filter_policy: bloomfilter 2024-07-15T06:56:57.382 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: whole_key_filtering: 1 2024-07-15T06:56:57.383 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: verify_compression: 0 2024-07-15T06:56:57.383 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: read_amp_bytes_per_bit: 0 2024-07-15T06:56:57.383 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: format_version: 5 2024-07-15T06:56:57.383 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: enable_index_compression: 1 2024-07-15T06:56:57.383 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: block_align: 0 2024-07-15T06:56:57.383 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: max_auto_readahead_size: 262144 2024-07-15T06:56:57.383 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: prepopulate_block_cache: 0 2024-07-15T06:56:57.383 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: initial_auto_readahead_size: 8192 2024-07-15T06:56:57.383 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: num_file_reads_for_auto_readahead: 2 2024-07-15T06:56:57.383 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.write_buffer_size: 33554432 2024-07-15T06:56:57.383 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.max_write_buffer_number: 2 2024-07-15T06:56:57.385 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.compression: NoCompression 2024-07-15T06:56:57.385 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.bottommost_compression: Disabled 2024-07-15T06:56:57.385 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.prefix_extractor: nullptr 2024-07-15T06:56:57.385 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-07-15T06:56:57.385 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.num_levels: 7 2024-07-15T06:56:57.385 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-07-15T06:56:57.385 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-07-15T06:56:57.385 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-07-15T06:56:57.385 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-07-15T06:56:57.385 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2024-07-15T06:56:57.385 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-07-15T06:56:57.385 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-07-15T06:56:57.386 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-07-15T06:56:57.386 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-07-15T06:56:57.386 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2024-07-15T06:56:57.386 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-07-15T06:56:57.386 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-07-15T06:56:57.386 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.compression_opts.window_bits: -14 2024-07-15T06:56:57.386 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.compression_opts.level: 32767 2024-07-15T06:56:57.386 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.compression_opts.strategy: 0 2024-07-15T06:56:57.386 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-07-15T06:56:57.386 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-07-15T06:56:57.386 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-07-15T06:56:57.386 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.compression_opts.parallel_threads: 1 2024-07-15T06:56:57.387 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.compression_opts.enabled: false 2024-07-15T06:56:57.387 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-07-15T06:56:57.387 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-07-15T06:56:57.387 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-07-15T06:56:57.387 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.level0_stop_writes_trigger: 36 2024-07-15T06:56:57.387 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.target_file_size_base: 67108864 2024-07-15T06:56:57.387 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.target_file_size_multiplier: 1 2024-07-15T06:56:57.387 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2024-07-15T06:56:57.387 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-07-15T06:56:57.387 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-07-15T06:56:57.387 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-07-15T06:56:57.387 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-07-15T06:56:57.387 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-07-15T06:56:57.387 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-07-15T06:56:57.388 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-07-15T06:56:57.388 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-07-15T06:56:57.388 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-07-15T06:56:57.388 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-07-15T06:56:57.388 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.max_compaction_bytes: 1677721600 2024-07-15T06:56:57.388 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-07-15T06:56:57.388 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.arena_block_size: 1048576 2024-07-15T06:56:57.388 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-07-15T06:56:57.388 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-07-15T06:56:57.388 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.disable_auto_compactions: 0 2024-07-15T06:56:57.388 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-07-15T06:56:57.388 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-07-15T06:56:57.388 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-07-15T06:56:57.389 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-07-15T06:56:57.389 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-07-15T06:56:57.389 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-07-15T06:56:57.389 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-07-15T06:56:57.389 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-07-15T06:56:57.389 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-07-15T06:56:57.389 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-07-15T06:56:57.389 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.table_properties_collectors: 2024-07-15T06:56:57.389 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.inplace_update_support: 0 2024-07-15T06:56:57.389 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.inplace_update_num_locks: 10000 2024-07-15T06:56:57.389 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-07-15T06:56:57.389 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.memtable_whole_key_filtering: 0 2024-07-15T06:56:57.389 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.memtable_huge_page_size: 0 2024-07-15T06:56:57.390 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.bloom_locality: 0 2024-07-15T06:56:57.390 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.max_successive_merges: 0 2024-07-15T06:56:57.390 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.optimize_filters_for_hits: 0 2024-07-15T06:56:57.390 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.paranoid_file_checks: 0 2024-07-15T06:56:57.390 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.force_consistency_checks: 1 2024-07-15T06:56:57.390 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.report_bg_io_stats: 0 2024-07-15T06:56:57.390 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.ttl: 2592000 2024-07-15T06:56:57.390 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.periodic_compaction_seconds: 0 2024-07-15T06:56:57.390 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.preclude_last_level_data_seconds: 0 2024-07-15T06:56:57.390 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.preserve_internal_time_seconds: 0 2024-07-15T06:56:57.390 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.enable_blob_files: false 2024-07-15T06:56:57.390 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.min_blob_size: 0 2024-07-15T06:56:57.391 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.blob_file_size: 268435456 2024-07-15T06:56:57.391 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.blob_compression_type: NoCompression 2024-07-15T06:56:57.391 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.enable_blob_garbage_collection: false 2024-07-15T06:56:57.391 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-07-15T06:56:57.391 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-07-15T06:56:57.391 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.blob_compaction_readahead_size: 0 2024-07-15T06:56:57.391 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.blob_file_starting_level: 0 2024-07-15T06:56:57.391 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-07-15T06:56:57.391 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb56420b640 3 rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 42.sst 2024-07-15T06:56:57.391 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-07-15T06:56:57.391 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 44, last_sequence is 16972, log_number is 40,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-07-15T06:56:57.391 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 40 2024-07-15T06:56:57.391 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 8bedf614-7361-410b-bc7e-ccbb3003ec0a 2024-07-15T06:56:57.392 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1721026617209744, "job": 1, "event": "recovery_started", "wal_files": [40]} 2024-07-15T06:56:57.392 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #40 mode 2 2024-07-15T06:56:57.392 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.203+0000 7fb568434b00 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1721026617211209, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 45, "file_size": 134241, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 16973, "largest_seqno": 17131, "table_properties": {"data_size": 132537, "index_size": 436, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 261, "raw_key_size": 2474, "raw_average_key_size": 27, "raw_value_size": 130524, "raw_average_value_size": 1450, "num_data_blocks": 18, "num_entries": 90, "num_filter_entries": 90, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1721026617, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "8bedf614-7361-410b-bc7e-ccbb3003ec0a", "db_session_id": "MLJ68VVVPYC33U9MPK23", "orig_file_number": 45, "seqno_to_time_mapping": "N/A"}} 2024-07-15T06:56:57.392 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.207+0000 7fb568434b00 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1721026617211332, "job": 1, "event": "recovery_finished"} 2024-07-15T06:56:57.392 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.207+0000 7fb568434b00 4 rocksdb: [db/version_set.cc:5047] Creating manifest 47 2024-07-15T06:56:57.392 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.207+0000 7fb568434b00 4 rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-07-15T06:56:57.392 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.207+0000 7fb568434b00 4 rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000040.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-07-15T06:56:57.392 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.207+0000 7fb568434b00 4 rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55a7357c0e00 2024-07-15T06:56:57.392 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.207+0000 7fb568434b00 4 rocksdb: DB pointer 0x55a7358b8000 2024-07-15T06:56:57.392 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.207+0000 7fb55e1ff640 4 rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-07-15T06:56:57.392 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.207+0000 7fb55e1ff640 4 rocksdb: [db/db_impl/db_impl.cc:1111] 2024-07-15T06:56:57.392 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: ** DB Stats ** 2024-07-15T06:56:57.392 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: Uptime(secs): 0.0 total, 0.0 interval 2024-07-15T06:56:57.393 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-07-15T06:56:57.393 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-07-15T06:56:57.393 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-07-15T06:56:57.393 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-07-15T06:56:57.393 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-07-15T06:56:57.393 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-07-15T06:56:57.393 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: ** Compaction Stats [default] ** 2024-07-15T06:56:57.393 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-07-15T06:56:57.393 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-07-15T06:56:57.393 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: L0 1/0 131.09 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 123.0 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-07-15T06:56:57.393 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: L6 1/0 7.17 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-07-15T06:56:57.393 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: Sum 2/0 7.30 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 123.0 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-07-15T06:56:57.394 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 123.0 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-07-15T06:56:57.394 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: ** Compaction Stats [default] ** 2024-07-15T06:56:57.394 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-07-15T06:56:57.394 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-07-15T06:56:57.394 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 123.0 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-07-15T06:56:57.394 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-07-15T06:56:57.394 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: Uptime(secs): 0.0 total, 0.0 interval 2024-07-15T06:56:57.394 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: Flush(GB): cumulative 0.000, interval 0.000 2024-07-15T06:56:57.394 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: AddFile(GB): cumulative 0.000, interval 0.000 2024-07-15T06:56:57.394 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: AddFile(Total Files): cumulative 0, interval 0 2024-07-15T06:56:57.394 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: AddFile(L0 Files): cumulative 0, interval 0 2024-07-15T06:56:57.394 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: AddFile(Keys): cumulative 0, interval 0 2024-07-15T06:56:57.394 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: Cumulative compaction: 0.00 GB write, 28.51 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-07-15T06:56:57.395 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: Interval compaction: 0.00 GB write, 28.51 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-07-15T06:56:57.395 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-07-15T06:56:57.395 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: Block cache BinnedLRUCache@0x55a7357951f0#7 capacity: 512.00 MB usage: 0.83 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.6e-05 secs_since: 0 2024-07-15T06:56:57.395 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: Block cache entry stats(count,size,portion): FilterBlock(1,0.31 KB,5.96046e-05%) IndexBlock(1,0.52 KB,9.83477e-05%) Misc(1,0.00 KB,0%) 2024-07-15T06:56:57.395 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: ** File Read Latency Histogram By Level [default] ** 2024-07-15T06:56:57.395 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.207+0000 7fb568434b00 0 starting mon.a rank 0 at public addrs [v2:172.21.15.89:3300/0,v1:172.21.15.89:6789/0] at bind addrs [v2:172.21.15.89:3300/0,v1:172.21.15.89:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 75ba02bc-4274-11ef-bcad-c7b262605968 2024-07-15T06:56:57.395 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.211+0000 7fb568434b00 1 mon.a@-1(???) e3 preinit fsid 75ba02bc-4274-11ef-bcad-c7b262605968 2024-07-15T06:56:57.395 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.211+0000 7fb568434b00 0 mon.a@-1(???).mds e10 new map 2024-07-15T06:56:57.395 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.211+0000 7fb568434b00 0 mon.a@-1(???).mds e10 print_map 2024-07-15T06:56:57.395 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: e10 2024-07-15T06:56:57.395 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: enable_multiple, ever_enabled_multiple: 1,1 2024-07-15T06:56:57.395 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-07-15T06:56:57.395 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: legacy client fscid: 1 2024-07-15T06:56:57.395 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: 2024-07-15T06:56:57.395 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: Filesystem 'foo' (1) 2024-07-15T06:56:57.396 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: fs_name foo 2024-07-15T06:56:57.396 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: epoch 10 2024-07-15T06:56:57.396 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: flags 12 joinable allow_snaps allow_multimds_snaps 2024-07-15T06:56:57.396 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: created 2024-07-15T06:50:29.283802+0000 2024-07-15T06:56:57.396 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: modified 2024-07-15T06:50:39.720043+0000 2024-07-15T06:56:57.396 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: tableserver 0 2024-07-15T06:56:57.396 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: root 0 2024-07-15T06:56:57.396 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: session_timeout 60 2024-07-15T06:56:57.396 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: session_autoclose 300 2024-07-15T06:56:57.396 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: max_file_size 1099511627776 2024-07-15T06:56:57.396 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: max_xattr_size 65536 2024-07-15T06:56:57.396 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: required_client_features {} 2024-07-15T06:56:57.396 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: last_failure 0 2024-07-15T06:56:57.396 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: last_failure_osd_epoch 68 2024-07-15T06:56:57.397 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-07-15T06:56:57.397 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: max_mds 1 2024-07-15T06:56:57.397 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: in 0 2024-07-15T06:56:57.397 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: up {0=24415} 2024-07-15T06:56:57.397 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: failed 2024-07-15T06:56:57.397 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: damaged 2024-07-15T06:56:57.397 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: stopped 2024-07-15T06:56:57.397 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: data_pools [4] 2024-07-15T06:56:57.397 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: metadata_pool 3 2024-07-15T06:56:57.397 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: inline_data disabled 2024-07-15T06:56:57.397 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: balancer 2024-07-15T06:56:57.397 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: bal_rank_mask -1 2024-07-15T06:56:57.397 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: standby_count_wanted 1 2024-07-15T06:56:57.398 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: [mds.foo.smithi134.wugtzh{0:24415} state up:active seq 4 join_fscid=1 addr [v2:172.21.15.134:6832/968641534,v1:172.21.15.134:6833/968641534] compat {c=[1],r=[1],i=[7ff]}] 2024-07-15T06:56:57.398 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: 2024-07-15T06:56:57.398 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: 2024-07-15T06:56:57.398 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: Standby daemons: 2024-07-15T06:56:57.398 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: 2024-07-15T06:56:57.398 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: [mds.foo.smithi089.cczuoz{-1:24488} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.89:6834/1598068998,v1:172.21.15.89:6835/1598068998] compat {c=[1],r=[1],i=[7ff]}] 2024-07-15T06:56:57.398 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.211+0000 7fb568434b00 0 mon.a@-1(???).osd e78 crush map has features 3314933000854323200, adjusting msgr requires 2024-07-15T06:56:57.398 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.211+0000 7fb568434b00 0 mon.a@-1(???).osd e78 crush map has features 432629239337189376, adjusting msgr requires 2024-07-15T06:56:57.398 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.211+0000 7fb568434b00 0 mon.a@-1(???).osd e78 crush map has features 432629239337189376, adjusting msgr requires 2024-07-15T06:56:57.398 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.211+0000 7fb568434b00 0 mon.a@-1(???).osd e78 crush map has features 432629239337189376, adjusting msgr requires 2024-07-15T06:56:57.398 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:57 smithi089 bash[79810]: debug 2024-07-15T06:56:57.211+0000 7fb568434b00 1 mon.a@-1(???).paxosservice(auth 1..20) refresh upgraded, format 0 -> 3 2024-07-15T06:56:59.053 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:56:58 smithi134 bash[60729]: ts=2024-07-15T06:56:58.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1102-g9a202024 (9a2020246f85463d4fdb115c94aa40b463caa1d2) quincy (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1102-g9a202024 (9a2020246f85463d4fdb115c94aa40b463caa1d2) quincy (stable)\", cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:56:59.053 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:56:58 smithi134 bash[61128]: ignoring --setuser ceph since I am not root 2024-07-15T06:56:59.053 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:56:58 smithi134 bash[61128]: ignoring --setgroup ceph since I am not root 2024-07-15T06:56:59.054 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:56:58 smithi134 bash[61128]: debug 2024-07-15T06:56:58.769+0000 7fbabe8f7640 1 -- 172.21.15.134:0/618723446 <== mon.1 v2:172.21.15.134:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x561486d94d00 con 0x561486d87800 2024-07-15T06:56:59.054 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:56:58 smithi134 bash[61128]: debug 2024-07-15T06:56:58.881+0000 7fbac1152040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-07-15T06:56:59.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:58 smithi134 bash[24068]: cluster 2024-07-15T06:56:54.972435+0000 mgr.y (mgr.25016) 49 : cluster [DBG] pgmap v29: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:56:59.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:58 smithi134 bash[24068]: audit 2024-07-15T06:56:56.904180+0000 mon.c (mon.2) 69 : audit [DBG] from='client.? 172.21.15.89:0/2726370392' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:56:59.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:58 smithi134 bash[24068]: cluster 2024-07-15T06:56:56.973034+0000 mgr.y (mgr.25016) 50 : cluster [DBG] pgmap v30: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:56:59.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:58 smithi134 bash[24068]: cluster 2024-07-15T06:56:57.833349+0000 mon.a (mon.0) 1 : cluster [INF] mon.a calling monitor election 2024-07-15T06:56:59.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:58 smithi134 bash[24068]: cluster 2024-07-15T06:56:58.657802+0000 mon.a (mon.0) 2 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-07-15T06:56:59.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:58 smithi134 bash[24068]: cluster 2024-07-15T06:56:58.669807+0000 mon.a (mon.0) 3 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.89:3300/0,v1:172.21.15.89:6789/0],b=[v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0],c=[v2:172.21.15.89:3301/0,v1:172.21.15.89:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-07-15T06:56:59.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:58 smithi134 bash[24068]: cluster 2024-07-15T06:56:58.671044+0000 mon.a (mon.0) 4 : cluster [DBG] fsmap foo:1 {0=foo.smithi134.wugtzh=up:active} 1 up:standby 2024-07-15T06:56:59.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:58 smithi134 bash[24068]: cluster 2024-07-15T06:56:58.671145+0000 mon.a (mon.0) 5 : cluster [DBG] osdmap e78: 8 total, 8 up, 8 in 2024-07-15T06:56:59.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:58 smithi134 bash[24068]: cluster 2024-07-15T06:56:58.673417+0000 mon.a (mon.0) 6 : cluster [DBG] mgrmap e34: y(active, since 55s), standbys: x 2024-07-15T06:56:59.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:58 smithi134 bash[24068]: cluster 2024-07-15T06:56:58.674436+0000 mon.a (mon.0) 7 : cluster [INF] overall HEALTH_OK 2024-07-15T06:56:59.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:58 smithi134 bash[24068]: cluster 2024-07-15T06:56:58.708674+0000 mon.a (mon.0) 8 : cluster [DBG] mgrmap e35: y(active, since 55s), standbys: x 2024-07-15T06:56:59.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:58 smithi134 bash[24068]: audit 2024-07-15T06:56:58.708954+0000 mon.a (mon.0) 9 : audit [INF] from='mgr.25016 ' entity='' 2024-07-15T06:56:59.119 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:56:58 smithi089 bash[72894]: ignoring --setuser ceph since I am not root 2024-07-15T06:56:59.120 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:56:58 smithi089 bash[72894]: ignoring --setgroup ceph since I am not root 2024-07-15T06:56:59.120 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:56:58 smithi089 bash[72894]: debug 2024-07-15T06:56:58.771+0000 7f1183621640 1 -- 172.21.15.89:0/1900029318 <== mon.2 v2:172.21.15.89:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55b93ac58d00 con 0x55b93ac4b800 2024-07-15T06:56:59.120 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:56:58 smithi089 bash[72894]: debug 2024-07-15T06:56:58.887+0000 7f1185e7c040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-07-15T06:56:59.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:58 smithi089 bash[79810]: cluster 2024-07-15T06:56:54.972435+0000 mgr.y (mgr.25016) 49 : cluster [DBG] pgmap v29: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:56:59.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:58 smithi089 bash[79810]: audit 2024-07-15T06:56:56.904180+0000 mon.c (mon.2) 69 : audit [DBG] from='client.? 172.21.15.89:0/2726370392' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:56:59.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:58 smithi089 bash[79810]: cluster 2024-07-15T06:56:56.973034+0000 mgr.y (mgr.25016) 50 : cluster [DBG] pgmap v30: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:56:59.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:58 smithi089 bash[79810]: cluster 2024-07-15T06:56:57.833349+0000 mon.a (mon.0) 1 : cluster [INF] mon.a calling monitor election 2024-07-15T06:56:59.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:58 smithi089 bash[79810]: cluster 2024-07-15T06:56:58.657802+0000 mon.a (mon.0) 2 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-07-15T06:56:59.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:58 smithi089 bash[79810]: cluster 2024-07-15T06:56:58.669807+0000 mon.a (mon.0) 3 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.89:3300/0,v1:172.21.15.89:6789/0],b=[v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0],c=[v2:172.21.15.89:3301/0,v1:172.21.15.89:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-07-15T06:56:59.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:58 smithi089 bash[79810]: cluster 2024-07-15T06:56:58.671044+0000 mon.a (mon.0) 4 : cluster [DBG] fsmap foo:1 {0=foo.smithi134.wugtzh=up:active} 1 up:standby 2024-07-15T06:56:59.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:58 smithi089 bash[79810]: cluster 2024-07-15T06:56:58.671145+0000 mon.a (mon.0) 5 : cluster [DBG] osdmap e78: 8 total, 8 up, 8 in 2024-07-15T06:56:59.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:58 smithi089 bash[79810]: cluster 2024-07-15T06:56:58.673417+0000 mon.a (mon.0) 6 : cluster [DBG] mgrmap e34: y(active, since 55s), standbys: x 2024-07-15T06:56:59.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:58 smithi089 bash[79810]: cluster 2024-07-15T06:56:58.674436+0000 mon.a (mon.0) 7 : cluster [INF] overall HEALTH_OK 2024-07-15T06:56:59.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:58 smithi089 bash[79810]: cluster 2024-07-15T06:56:58.708674+0000 mon.a (mon.0) 8 : cluster [DBG] mgrmap e35: y(active, since 55s), standbys: x 2024-07-15T06:56:59.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:58 smithi089 bash[79810]: audit 2024-07-15T06:56:58.708954+0000 mon.a (mon.0) 9 : audit [INF] from='mgr.25016 ' entity='' 2024-07-15T06:56:59.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:58 smithi089 bash[31372]: cluster 2024-07-15T06:56:54.972435+0000 mgr.y (mgr.25016) 49 : cluster [DBG] pgmap v29: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:56:59.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:58 smithi089 bash[31372]: audit 2024-07-15T06:56:56.904180+0000 mon.c (mon.2) 69 : audit [DBG] from='client.? 172.21.15.89:0/2726370392' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:56:59.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:58 smithi089 bash[31372]: cluster 2024-07-15T06:56:56.973034+0000 mgr.y (mgr.25016) 50 : cluster [DBG] pgmap v30: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:56:59.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:58 smithi089 bash[31372]: cluster 2024-07-15T06:56:57.833349+0000 mon.a (mon.0) 1 : cluster [INF] mon.a calling monitor election 2024-07-15T06:56:59.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:58 smithi089 bash[31372]: cluster 2024-07-15T06:56:58.657802+0000 mon.a (mon.0) 2 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-07-15T06:56:59.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:58 smithi089 bash[31372]: cluster 2024-07-15T06:56:58.669807+0000 mon.a (mon.0) 3 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.89:3300/0,v1:172.21.15.89:6789/0],b=[v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0],c=[v2:172.21.15.89:3301/0,v1:172.21.15.89:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-07-15T06:56:59.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:58 smithi089 bash[31372]: cluster 2024-07-15T06:56:58.671044+0000 mon.a (mon.0) 4 : cluster [DBG] fsmap foo:1 {0=foo.smithi134.wugtzh=up:active} 1 up:standby 2024-07-15T06:56:59.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:58 smithi089 bash[31372]: cluster 2024-07-15T06:56:58.671145+0000 mon.a (mon.0) 5 : cluster [DBG] osdmap e78: 8 total, 8 up, 8 in 2024-07-15T06:56:59.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:58 smithi089 bash[31372]: cluster 2024-07-15T06:56:58.673417+0000 mon.a (mon.0) 6 : cluster [DBG] mgrmap e34: y(active, since 55s), standbys: x 2024-07-15T06:56:59.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:58 smithi089 bash[31372]: cluster 2024-07-15T06:56:58.674436+0000 mon.a (mon.0) 7 : cluster [INF] overall HEALTH_OK 2024-07-15T06:56:59.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:58 smithi089 bash[31372]: cluster 2024-07-15T06:56:58.708674+0000 mon.a (mon.0) 8 : cluster [DBG] mgrmap e35: y(active, since 55s), standbys: x 2024-07-15T06:56:59.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:58 smithi089 bash[31372]: audit 2024-07-15T06:56:58.708954+0000 mon.a (mon.0) 9 : audit [INF] from='mgr.25016 ' entity='' 2024-07-15T06:56:59.258 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 18.2.2-1609-g6ea7aaa9 (6ea7aaa9f93d34dc4d21862afe00140dfddbd706) reef (stable)": 1 2024-07-15T06:57:00.053 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:56:59 smithi134 bash[24068]: audit 2024-07-15T06:56:59.251971+0000 mon.c (mon.2) 70 : audit [DBG] from='client.? 172.21.15.89:0/2474105274' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:57:00.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:56:59 smithi089 bash[31372]: audit 2024-07-15T06:56:59.251971+0000 mon.c (mon.2) 70 : audit [DBG] from='client.? 172.21.15.89:0/2474105274' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:57:00.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:56:59 smithi089 bash[79810]: audit 2024-07-15T06:56:59.251971+0000 mon.c (mon.2) 70 : audit [DBG] from='client.? 172.21.15.89:0/2474105274' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:57:00.138 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -e sha1=6ea7aaa9f93d34dc4d21862afe00140dfddbd706 -- bash -c 'ceph orch ps' 2024-07-15T06:57:00.553 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:57:00 smithi134 bash[61128]: debug 2024-07-15T06:57:00.117+0000 7fbac1152040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-07-15T06:57:00.553 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:57:00 smithi134 bash[61128]: debug 2024-07-15T06:57:00.185+0000 7fbac1152040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-07-15T06:57:00.553 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:57:00 smithi134 bash[61128]: debug 2024-07-15T06:57:00.341+0000 7fbac1152040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-07-15T06:57:00.619 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:57:00 smithi089 bash[72894]: debug 2024-07-15T06:57:00.119+0000 7f1185e7c040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-07-15T06:57:00.619 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:57:00 smithi089 bash[72894]: debug 2024-07-15T06:57:00.191+0000 7f1185e7c040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-07-15T06:57:00.619 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:57:00 smithi089 bash[72894]: debug 2024-07-15T06:57:00.351+0000 7f1185e7c040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-07-15T06:57:01.303 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:57:01 smithi134 bash[61128]: debug 2024-07-15T06:57:00.993+0000 7fbac1152040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-07-15T06:57:01.303 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:57:01 smithi134 bash[61128]: debug 2024-07-15T06:57:01.065+0000 7fbac1152040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-07-15T06:57:01.303 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:57:01 smithi134 bash[61128]: debug 2024-07-15T06:57:01.133+0000 7fbac1152040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-07-15T06:57:01.336 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:57:01 smithi089 bash[72894]: debug 2024-07-15T06:57:01.003+0000 7f1185e7c040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-07-15T06:57:01.336 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:57:01 smithi089 bash[72894]: debug 2024-07-15T06:57:01.071+0000 7f1185e7c040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-07-15T06:57:01.336 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:57:01 smithi089 bash[72894]: debug 2024-07-15T06:57:01.143+0000 7f1185e7c040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-07-15T06:57:01.619 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:57:01 smithi089 bash[72894]: debug 2024-07-15T06:57:01.335+0000 7f1185e7c040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-07-15T06:57:01.620 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:57:01 smithi089 bash[72894]: debug 2024-07-15T06:57:01.415+0000 7f1185e7c040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-07-15T06:57:01.620 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:57:01 smithi089 bash[72894]: debug 2024-07-15T06:57:01.483+0000 7f1185e7c040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-07-15T06:57:01.731 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:57:01 smithi134 bash[61128]: debug 2024-07-15T06:57:01.325+0000 7fbac1152040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-07-15T06:57:01.732 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:57:01 smithi134 bash[61128]: debug 2024-07-15T06:57:01.405+0000 7fbac1152040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-07-15T06:57:01.732 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:57:01 smithi134 bash[61128]: debug 2024-07-15T06:57:01.473+0000 7fbac1152040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-07-15T06:57:02.002 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:57:01 smithi134 bash[61128]: debug 2024-07-15T06:57:01.733+0000 7fbac1152040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-07-15T06:57:02.002 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:57:01 smithi134 bash[61128]: debug 2024-07-15T06:57:01.805+0000 7fbac1152040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-07-15T06:57:02.014 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:57:01 smithi089 bash[72894]: debug 2024-07-15T06:57:01.743+0000 7f1185e7c040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-07-15T06:57:02.014 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:57:01 smithi089 bash[72894]: debug 2024-07-15T06:57:01.815+0000 7f1185e7c040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-07-15T06:57:02.303 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:57:02 smithi134 bash[61128]: debug 2024-07-15T06:57:02.001+0000 7fbac1152040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-07-15T06:57:02.303 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:57:02 smithi134 bash[61128]: debug 2024-07-15T06:57:02.081+0000 7fbac1152040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-07-15T06:57:02.344 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:57:02 smithi089 bash[72894]: debug 2024-07-15T06:57:02.015+0000 7f1185e7c040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-07-15T06:57:02.344 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:57:02 smithi089 bash[72894]: debug 2024-07-15T06:57:02.091+0000 7f1185e7c040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-07-15T06:57:02.619 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:57:02 smithi089 bash[72894]: debug 2024-07-15T06:57:02.343+0000 7f1185e7c040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-07-15T06:57:02.619 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:57:02 smithi089 bash[72894]: debug 2024-07-15T06:57:02.523+0000 7f1185e7c040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-07-15T06:57:02.678 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:57:02 smithi134 bash[60729]: ts=2024-07-15T06:57:02.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:57:02.679 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:57:02 smithi134 bash[61128]: debug 2024-07-15T06:57:02.333+0000 7fbac1152040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-07-15T06:57:02.679 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:57:02 smithi134 bash[61128]: debug 2024-07-15T06:57:02.513+0000 7fbac1152040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-07-15T06:57:02.939 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:57:02 smithi134 bash[61128]: debug 2024-07-15T06:57:02.677+0000 7fbac1152040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-07-15T06:57:02.939 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:57:02 smithi134 bash[61128]: debug 2024-07-15T06:57:02.757+0000 7fbac1152040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-07-15T06:57:02.953 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:57:02 smithi089 bash[72894]: debug 2024-07-15T06:57:02.687+0000 7f1185e7c040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-07-15T06:57:02.953 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:57:02 smithi089 bash[72894]: debug 2024-07-15T06:57:02.767+0000 7f1185e7c040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-07-15T06:57:03.303 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:57:02 smithi134 bash[61128]: debug 2024-07-15T06:57:02.941+0000 7fbac1152040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-07-15T06:57:03.369 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:57:02 smithi089 bash[72894]: debug 2024-07-15T06:57:02.951+0000 7f1185e7c040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-07-15T06:57:03.774 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:57:03 smithi134 bash[60729]: ts=2024-07-15T06:57:03.505Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi134\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi134\", job=\"node\", machine=\"x86_64\", nodename=\"smithi134\", release=\"5.15.0-113-generic\", sysname=\"Linux\", version=\"#123-Ubuntu SMP Mon Jun 10 08:16:17 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", domainname=\"(none)\", instance=\"smithi134\", job=\"node\", machine=\"x86_64\", nodename=\"smithi134\", release=\"5.15.0-113-generic\", sysname=\"Linux\", version=\"#123-Ubuntu SMP Mon Jun 10 08:16:17 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:57:04.053 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:57:03 smithi134 bash[61128]: debug 2024-07-15T06:57:03.773+0000 7fbac1152040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-07-15T06:57:04.053 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:57:03 smithi134 bash[61128]: debug 2024-07-15T06:57:03.853+0000 7fbac1152040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-07-15T06:57:04.100 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:57:03 smithi089 bash[72894]: debug 2024-07-15T06:57:03.795+0000 7f1185e7c040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-07-15T06:57:04.100 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:57:03 smithi089 bash[72894]: debug 2024-07-15T06:57:03.871+0000 7f1185e7c040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-07-15T06:57:04.370 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:57:04 smithi089 bash[72894]: debug 2024-07-15T06:57:04.099+0000 7f1185e7c040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-07-15T06:57:04.370 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:57:04 smithi089 bash[72894]: debug 2024-07-15T06:57:04.175+0000 7f1185e7c040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-07-15T06:57:04.546 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:57:04 smithi134 bash[61128]: debug 2024-07-15T06:57:04.081+0000 7fbac1152040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-07-15T06:57:04.546 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:57:04 smithi134 bash[61128]: debug 2024-07-15T06:57:04.153+0000 7fbac1152040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-07-15T06:57:04.547 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:57:04 smithi134 bash[61128]: debug 2024-07-15T06:57:04.545+0000 7fbac1152040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-07-15T06:57:04.803 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:57:04 smithi134 bash[61128]: debug 2024-07-15T06:57:04.629+0000 7fbac1152040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-07-15T06:57:04.803 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:57:04 smithi134 bash[61128]: [15/Jul/2024:06:57:04] ENGINE Bus STARTING 2024-07-15T06:57:04.803 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:57:04 smithi134 bash[61128]: CherryPy Checker: 2024-07-15T06:57:04.803 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:57:04 smithi134 bash[61128]: The Application mounted at '' has an empty config. 2024-07-15T06:57:04.803 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:57:04 smithi134 bash[61128]: [15/Jul/2024:06:57:04] ENGINE Serving on http://:::9283 2024-07-15T06:57:04.803 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:57:04 smithi134 bash[61128]: [15/Jul/2024:06:57:04] ENGINE Bus STARTED 2024-07-15T06:57:04.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:04 smithi134 bash[24068]: cluster 2024-07-15T06:57:04.655841+0000 mon.a (mon.0) 10 : cluster [DBG] Standby manager daemon x restarted 2024-07-15T06:57:04.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:04 smithi134 bash[24068]: cluster 2024-07-15T06:57:04.655949+0000 mon.a (mon.0) 11 : cluster [DBG] Standby manager daemon x started 2024-07-15T06:57:04.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:04 smithi134 bash[24068]: audit 2024-07-15T06:57:04.661025+0000 mon.b (mon.1) 134 : audit [DBG] from='mgr.? 172.21.15.134:0/447531343' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-07-15T06:57:04.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:04 smithi134 bash[24068]: audit 2024-07-15T06:57:04.662061+0000 mon.b (mon.1) 135 : audit [DBG] from='mgr.? 172.21.15.134:0/447531343' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-07-15T06:57:04.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:04 smithi134 bash[24068]: audit 2024-07-15T06:57:04.663394+0000 mon.b (mon.1) 136 : audit [DBG] from='mgr.? 172.21.15.134:0/447531343' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-07-15T06:57:04.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:04 smithi134 bash[24068]: audit 2024-07-15T06:57:04.664270+0000 mon.b (mon.1) 137 : audit [DBG] from='mgr.? 172.21.15.134:0/447531343' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-07-15T06:57:04.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:04 smithi134 bash[24068]: cluster 2024-07-15T06:57:04.674886+0000 mon.a (mon.0) 12 : cluster [INF] Active manager daemon y restarted 2024-07-15T06:57:04.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:04 smithi134 bash[24068]: cluster 2024-07-15T06:57:04.675455+0000 mon.a (mon.0) 13 : cluster [ERR] Health check failed: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-07-15T06:57:04.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:04 smithi134 bash[24068]: cluster 2024-07-15T06:57:04.675711+0000 mon.a (mon.0) 14 : cluster [INF] Activating manager daemon y 2024-07-15T06:57:04.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:04 smithi134 bash[24068]: cluster 2024-07-15T06:57:04.689868+0000 mon.a (mon.0) 15 : cluster [DBG] osdmap e79: 8 total, 8 up, 8 in 2024-07-15T06:57:04.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:04 smithi134 bash[24068]: cluster 2024-07-15T06:57:04.690601+0000 mon.a (mon.0) 16 : cluster [DBG] mgrmap e36: y(active, starting, since 0.0150696s), standbys: x 2024-07-15T06:57:04.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:04 smithi134 bash[24068]: audit 2024-07-15T06:57:04.692823+0000 mon.c (mon.2) 71 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-07-15T06:57:04.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:04 smithi134 bash[24068]: audit 2024-07-15T06:57:04.693320+0000 mon.c (mon.2) 72 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-15T06:57:04.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:04 smithi134 bash[24068]: audit 2024-07-15T06:57:04.693938+0000 mon.c (mon.2) 73 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-15T06:57:04.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:04 smithi134 bash[24068]: audit 2024-07-15T06:57:04.694536+0000 mon.c (mon.2) 74 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi134.wugtzh"}]: dispatch 2024-07-15T06:57:04.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:04 smithi134 bash[24068]: audit 2024-07-15T06:57:04.695367+0000 mon.c (mon.2) 75 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi089.cczuoz"}]: dispatch 2024-07-15T06:57:04.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:04 smithi134 bash[24068]: audit 2024-07-15T06:57:04.696721+0000 mon.c (mon.2) 76 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-07-15T06:57:04.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:04 smithi134 bash[24068]: audit 2024-07-15T06:57:04.697303+0000 mon.c (mon.2) 77 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-07-15T06:57:04.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:04 smithi134 bash[24068]: audit 2024-07-15T06:57:04.697896+0000 mon.c (mon.2) 78 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-07-15T06:57:04.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:04 smithi134 bash[24068]: audit 2024-07-15T06:57:04.698295+0000 mon.c (mon.2) 79 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-07-15T06:57:04.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:04 smithi134 bash[24068]: audit 2024-07-15T06:57:04.698658+0000 mon.c (mon.2) 80 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-07-15T06:57:04.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:04 smithi134 bash[24068]: audit 2024-07-15T06:57:04.698944+0000 mon.c (mon.2) 81 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-07-15T06:57:04.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:04 smithi134 bash[24068]: audit 2024-07-15T06:57:04.699204+0000 mon.c (mon.2) 82 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-07-15T06:57:04.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:04 smithi134 bash[24068]: audit 2024-07-15T06:57:04.699496+0000 mon.c (mon.2) 83 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-07-15T06:57:04.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:04 smithi134 bash[24068]: audit 2024-07-15T06:57:04.699799+0000 mon.c (mon.2) 84 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-07-15T06:57:04.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:04 smithi134 bash[24068]: audit 2024-07-15T06:57:04.700127+0000 mon.c (mon.2) 85 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-07-15T06:57:04.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:04 smithi134 bash[24068]: audit 2024-07-15T06:57:04.700473+0000 mon.c (mon.2) 86 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-07-15T06:57:04.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:04 smithi134 bash[24068]: audit 2024-07-15T06:57:04.700840+0000 mon.c (mon.2) 87 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-07-15T06:57:04.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:04 smithi134 bash[24068]: audit 2024-07-15T06:57:04.701456+0000 mon.c (mon.2) 88 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-07-15T06:57:04.834 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:57:04 smithi089 bash[72894]: debug 2024-07-15T06:57:04.567+0000 7f1185e7c040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-07-15T06:57:04.835 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:57:04 smithi089 bash[72894]: debug 2024-07-15T06:57:04.647+0000 7f1185e7c040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-07-15T06:57:04.835 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:57:04 smithi089 bash[72894]: [15/Jul/2024:06:57:04] ENGINE Bus STARTING 2024-07-15T06:57:04.835 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:57:04 smithi089 bash[72894]: CherryPy Checker: 2024-07-15T06:57:04.835 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:57:04 smithi089 bash[72894]: The Application mounted at '' has an empty config. 2024-07-15T06:57:04.835 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:04 smithi089 bash[79810]: debug 2024-07-15T06:57:04.671+0000 7fb561205640 -1 log_channel(cluster) log [ERR] : Health check failed: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-07-15T06:57:04.836 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:04 smithi089 bash[79810]: cluster 2024-07-15T06:57:04.655841+0000 mon.a (mon.0) 10 : cluster [DBG] Standby manager daemon x restarted 2024-07-15T06:57:04.836 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:04 smithi089 bash[79810]: cluster 2024-07-15T06:57:04.655949+0000 mon.a (mon.0) 11 : cluster [DBG] Standby manager daemon x started 2024-07-15T06:57:04.836 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:04 smithi089 bash[79810]: audit 2024-07-15T06:57:04.661025+0000 mon.b (mon.1) 134 : audit [DBG] from='mgr.? 172.21.15.134:0/447531343' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-07-15T06:57:04.836 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:04 smithi089 bash[79810]: audit 2024-07-15T06:57:04.662061+0000 mon.b (mon.1) 135 : audit [DBG] from='mgr.? 172.21.15.134:0/447531343' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-07-15T06:57:04.836 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:04 smithi089 bash[79810]: audit 2024-07-15T06:57:04.663394+0000 mon.b (mon.1) 136 : audit [DBG] from='mgr.? 172.21.15.134:0/447531343' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-07-15T06:57:04.836 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:04 smithi089 bash[79810]: audit 2024-07-15T06:57:04.664270+0000 mon.b (mon.1) 137 : audit [DBG] from='mgr.? 172.21.15.134:0/447531343' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-07-15T06:57:04.836 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:04 smithi089 bash[79810]: cluster 2024-07-15T06:57:04.674886+0000 mon.a (mon.0) 12 : cluster [INF] Active manager daemon y restarted 2024-07-15T06:57:04.836 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:04 smithi089 bash[79810]: cluster 2024-07-15T06:57:04.675455+0000 mon.a (mon.0) 13 : cluster [ERR] Health check failed: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-07-15T06:57:04.836 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:04 smithi089 bash[79810]: cluster 2024-07-15T06:57:04.675711+0000 mon.a (mon.0) 14 : cluster [INF] Activating manager daemon y 2024-07-15T06:57:04.836 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:04 smithi089 bash[79810]: cluster 2024-07-15T06:57:04.689868+0000 mon.a (mon.0) 15 : cluster [DBG] osdmap e79: 8 total, 8 up, 8 in 2024-07-15T06:57:04.836 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:04 smithi089 bash[79810]: cluster 2024-07-15T06:57:04.690601+0000 mon.a (mon.0) 16 : cluster [DBG] mgrmap e36: y(active, starting, since 0.0150696s), standbys: x 2024-07-15T06:57:04.836 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:04 smithi089 bash[79810]: audit 2024-07-15T06:57:04.692823+0000 mon.c (mon.2) 71 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-07-15T06:57:04.836 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:04 smithi089 bash[79810]: audit 2024-07-15T06:57:04.693320+0000 mon.c (mon.2) 72 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-15T06:57:04.837 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:04 smithi089 bash[79810]: audit 2024-07-15T06:57:04.693938+0000 mon.c (mon.2) 73 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-15T06:57:04.837 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:04 smithi089 bash[79810]: audit 2024-07-15T06:57:04.694536+0000 mon.c (mon.2) 74 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi134.wugtzh"}]: dispatch 2024-07-15T06:57:04.837 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:04 smithi089 bash[79810]: audit 2024-07-15T06:57:04.695367+0000 mon.c (mon.2) 75 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi089.cczuoz"}]: dispatch 2024-07-15T06:57:04.837 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:04 smithi089 bash[79810]: audit 2024-07-15T06:57:04.696721+0000 mon.c (mon.2) 76 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-07-15T06:57:04.837 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:04 smithi089 bash[79810]: audit 2024-07-15T06:57:04.697303+0000 mon.c (mon.2) 77 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-07-15T06:57:04.837 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:04 smithi089 bash[79810]: audit 2024-07-15T06:57:04.697896+0000 mon.c (mon.2) 78 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-07-15T06:57:04.837 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:04 smithi089 bash[79810]: audit 2024-07-15T06:57:04.698295+0000 mon.c (mon.2) 79 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-07-15T06:57:04.837 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:04 smithi089 bash[79810]: audit 2024-07-15T06:57:04.698658+0000 mon.c (mon.2) 80 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-07-15T06:57:04.837 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:04 smithi089 bash[79810]: audit 2024-07-15T06:57:04.698944+0000 mon.c (mon.2) 81 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-07-15T06:57:04.837 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:04 smithi089 bash[79810]: audit 2024-07-15T06:57:04.699204+0000 mon.c (mon.2) 82 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-07-15T06:57:04.837 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:04 smithi089 bash[79810]: audit 2024-07-15T06:57:04.699496+0000 mon.c (mon.2) 83 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-07-15T06:57:04.837 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:04 smithi089 bash[79810]: audit 2024-07-15T06:57:04.699799+0000 mon.c (mon.2) 84 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-07-15T06:57:04.837 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:04 smithi089 bash[79810]: audit 2024-07-15T06:57:04.700127+0000 mon.c (mon.2) 85 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-07-15T06:57:04.838 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:04 smithi089 bash[79810]: audit 2024-07-15T06:57:04.700473+0000 mon.c (mon.2) 86 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-07-15T06:57:04.838 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:04 smithi089 bash[79810]: audit 2024-07-15T06:57:04.700840+0000 mon.c (mon.2) 87 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-07-15T06:57:04.838 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:04 smithi089 bash[79810]: audit 2024-07-15T06:57:04.701456+0000 mon.c (mon.2) 88 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-07-15T06:57:04.838 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:04 smithi089 bash[31372]: cluster 2024-07-15T06:57:04.655841+0000 mon.a (mon.0) 10 : cluster [DBG] Standby manager daemon x restarted 2024-07-15T06:57:04.838 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:04 smithi089 bash[31372]: cluster 2024-07-15T06:57:04.655949+0000 mon.a (mon.0) 11 : cluster [DBG] Standby manager daemon x started 2024-07-15T06:57:04.838 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:04 smithi089 bash[31372]: audit 2024-07-15T06:57:04.661025+0000 mon.b (mon.1) 134 : audit [DBG] from='mgr.? 172.21.15.134:0/447531343' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-07-15T06:57:04.838 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:04 smithi089 bash[31372]: audit 2024-07-15T06:57:04.662061+0000 mon.b (mon.1) 135 : audit [DBG] from='mgr.? 172.21.15.134:0/447531343' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-07-15T06:57:04.838 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:04 smithi089 bash[31372]: audit 2024-07-15T06:57:04.663394+0000 mon.b (mon.1) 136 : audit [DBG] from='mgr.? 172.21.15.134:0/447531343' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-07-15T06:57:04.838 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:04 smithi089 bash[31372]: audit 2024-07-15T06:57:04.664270+0000 mon.b (mon.1) 137 : audit [DBG] from='mgr.? 172.21.15.134:0/447531343' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-07-15T06:57:04.839 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:04 smithi089 bash[31372]: cluster 2024-07-15T06:57:04.674886+0000 mon.a (mon.0) 12 : cluster [INF] Active manager daemon y restarted 2024-07-15T06:57:04.839 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:04 smithi089 bash[31372]: cluster 2024-07-15T06:57:04.675455+0000 mon.a (mon.0) 13 : cluster [ERR] Health check failed: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-07-15T06:57:04.839 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:04 smithi089 bash[31372]: cluster 2024-07-15T06:57:04.675711+0000 mon.a (mon.0) 14 : cluster [INF] Activating manager daemon y 2024-07-15T06:57:04.839 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:04 smithi089 bash[31372]: cluster 2024-07-15T06:57:04.689868+0000 mon.a (mon.0) 15 : cluster [DBG] osdmap e79: 8 total, 8 up, 8 in 2024-07-15T06:57:04.839 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:04 smithi089 bash[31372]: cluster 2024-07-15T06:57:04.690601+0000 mon.a (mon.0) 16 : cluster [DBG] mgrmap e36: y(active, starting, since 0.0150696s), standbys: x 2024-07-15T06:57:04.839 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:04 smithi089 bash[31372]: audit 2024-07-15T06:57:04.692823+0000 mon.c (mon.2) 71 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-07-15T06:57:04.839 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:04 smithi089 bash[31372]: audit 2024-07-15T06:57:04.693320+0000 mon.c (mon.2) 72 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-15T06:57:04.839 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:04 smithi089 bash[31372]: audit 2024-07-15T06:57:04.693938+0000 mon.c (mon.2) 73 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-15T06:57:04.839 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:04 smithi089 bash[31372]: audit 2024-07-15T06:57:04.694536+0000 mon.c (mon.2) 74 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi134.wugtzh"}]: dispatch 2024-07-15T06:57:04.839 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:04 smithi089 bash[31372]: audit 2024-07-15T06:57:04.695367+0000 mon.c (mon.2) 75 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi089.cczuoz"}]: dispatch 2024-07-15T06:57:04.839 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:04 smithi089 bash[31372]: audit 2024-07-15T06:57:04.696721+0000 mon.c (mon.2) 76 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-07-15T06:57:04.839 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:04 smithi089 bash[31372]: audit 2024-07-15T06:57:04.697303+0000 mon.c (mon.2) 77 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-07-15T06:57:04.839 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:04 smithi089 bash[31372]: audit 2024-07-15T06:57:04.697896+0000 mon.c (mon.2) 78 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-07-15T06:57:04.840 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:04 smithi089 bash[31372]: audit 2024-07-15T06:57:04.698295+0000 mon.c (mon.2) 79 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-07-15T06:57:04.840 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:04 smithi089 bash[31372]: audit 2024-07-15T06:57:04.698658+0000 mon.c (mon.2) 80 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-07-15T06:57:04.840 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:04 smithi089 bash[31372]: audit 2024-07-15T06:57:04.698944+0000 mon.c (mon.2) 81 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-07-15T06:57:04.840 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:04 smithi089 bash[31372]: audit 2024-07-15T06:57:04.699204+0000 mon.c (mon.2) 82 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-07-15T06:57:04.840 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:04 smithi089 bash[31372]: audit 2024-07-15T06:57:04.699496+0000 mon.c (mon.2) 83 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-07-15T06:57:04.840 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:04 smithi089 bash[31372]: audit 2024-07-15T06:57:04.699799+0000 mon.c (mon.2) 84 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-07-15T06:57:04.840 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:04 smithi089 bash[31372]: audit 2024-07-15T06:57:04.700127+0000 mon.c (mon.2) 85 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-07-15T06:57:04.840 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:04 smithi089 bash[31372]: audit 2024-07-15T06:57:04.700473+0000 mon.c (mon.2) 86 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-07-15T06:57:04.840 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:04 smithi089 bash[31372]: audit 2024-07-15T06:57:04.700840+0000 mon.c (mon.2) 87 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-07-15T06:57:04.840 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:04 smithi089 bash[31372]: audit 2024-07-15T06:57:04.701456+0000 mon.c (mon.2) 88 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-07-15T06:57:04.903 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T06:57:05.120 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:57:04 smithi089 bash[72894]: debug 2024-07-15T06:57:04.835+0000 7f10f543c640 -1 client.0 error registering admin socket command: (17) File exists 2024-07-15T06:57:05.120 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:57:04 smithi089 bash[72894]: debug 2024-07-15T06:57:04.835+0000 7f10f543c640 -1 client.0 error registering admin socket command: (17) File exists 2024-07-15T06:57:05.120 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:57:04 smithi089 bash[72894]: debug 2024-07-15T06:57:04.835+0000 7f10f543c640 -1 client.0 error registering admin socket command: (17) File exists 2024-07-15T06:57:05.120 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:57:04 smithi089 bash[72894]: debug 2024-07-15T06:57:04.835+0000 7f10f543c640 -1 client.0 error registering admin socket command: (17) File exists 2024-07-15T06:57:05.120 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:57:04 smithi089 bash[72894]: debug 2024-07-15T06:57:04.835+0000 7f10f543c640 -1 client.0 error registering admin socket command: (17) File exists 2024-07-15T06:57:05.120 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:57:04 smithi089 bash[72894]: debug 2024-07-15T06:57:04.835+0000 7f10efc31640 -1 client.0 error registering admin socket command: (17) File exists 2024-07-15T06:57:05.120 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:57:04 smithi089 bash[72894]: debug 2024-07-15T06:57:04.835+0000 7f10efc31640 -1 client.0 error registering admin socket command: (17) File exists 2024-07-15T06:57:05.120 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:57:04 smithi089 bash[72894]: debug 2024-07-15T06:57:04.835+0000 7f10efc31640 -1 client.0 error registering admin socket command: (17) File exists 2024-07-15T06:57:05.120 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:57:04 smithi089 bash[72894]: debug 2024-07-15T06:57:04.835+0000 7f10efc31640 -1 client.0 error registering admin socket command: (17) File exists 2024-07-15T06:57:05.120 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:57:04 smithi089 bash[72894]: debug 2024-07-15T06:57:04.835+0000 7f10efc31640 -1 client.0 error registering admin socket command: (17) File exists 2024-07-15T06:57:05.120 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:57:04 smithi089 bash[72894]: [15/Jul/2024:06:57:04] ENGINE Serving on http://:::9283 2024-07-15T06:57:05.121 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:57:04 smithi089 bash[72894]: [15/Jul/2024:06:57:04] ENGINE Bus STARTED 2024-07-15T06:57:06.053 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:05 smithi134 bash[24068]: cluster 2024-07-15T06:57:04.726082+0000 mon.a (mon.0) 17 : cluster [INF] Manager daemon y is now available 2024-07-15T06:57:06.053 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:05 smithi134 bash[24068]: audit 2024-07-15T06:57:04.749337+0000 mon.c (mon.2) 89 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:57:06.053 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:05 smithi134 bash[24068]: audit 2024-07-15T06:57:04.777579+0000 mon.a (mon.0) 18 : audit [INF] from='mgr.25244 ' entity='mgr.y' 2024-07-15T06:57:06.053 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:05 smithi134 bash[24068]: audit 2024-07-15T06:57:04.779190+0000 mon.c (mon.2) 90 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:57:06.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:05 smithi134 bash[24068]: audit 2024-07-15T06:57:04.783642+0000 mon.c (mon.2) 91 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-07-15T06:57:06.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:05 smithi134 bash[24068]: audit 2024-07-15T06:57:04.783989+0000 mon.a (mon.0) 19 : audit [INF] from='mgr.25244 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-07-15T06:57:06.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:05 smithi134 bash[24068]: audit 2024-07-15T06:57:04.821432+0000 mon.c (mon.2) 92 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-07-15T06:57:06.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:05 smithi134 bash[24068]: audit 2024-07-15T06:57:04.821897+0000 mon.a (mon.0) 20 : audit [INF] from='mgr.25244 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-07-15T06:57:06.119 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:57:05 smithi089 bash[72894]: debug 2024-07-15T06:57:05.723+0000 7f112575c640 -1 mgr.server handle_report got status from non-daemon mon.a 2024-07-15T06:57:06.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:05 smithi089 bash[79810]: cluster 2024-07-15T06:57:04.726082+0000 mon.a (mon.0) 17 : cluster [INF] Manager daemon y is now available 2024-07-15T06:57:06.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:05 smithi089 bash[79810]: audit 2024-07-15T06:57:04.749337+0000 mon.c (mon.2) 89 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:57:06.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:05 smithi089 bash[79810]: audit 2024-07-15T06:57:04.777579+0000 mon.a (mon.0) 18 : audit [INF] from='mgr.25244 ' entity='mgr.y' 2024-07-15T06:57:06.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:05 smithi089 bash[79810]: audit 2024-07-15T06:57:04.779190+0000 mon.c (mon.2) 90 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:57:06.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:05 smithi089 bash[79810]: audit 2024-07-15T06:57:04.783642+0000 mon.c (mon.2) 91 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-07-15T06:57:06.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:05 smithi089 bash[79810]: audit 2024-07-15T06:57:04.783989+0000 mon.a (mon.0) 19 : audit [INF] from='mgr.25244 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-07-15T06:57:06.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:05 smithi089 bash[79810]: audit 2024-07-15T06:57:04.821432+0000 mon.c (mon.2) 92 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-07-15T06:57:06.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:05 smithi089 bash[79810]: audit 2024-07-15T06:57:04.821897+0000 mon.a (mon.0) 20 : audit [INF] from='mgr.25244 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-07-15T06:57:06.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:05 smithi089 bash[31372]: cluster 2024-07-15T06:57:04.726082+0000 mon.a (mon.0) 17 : cluster [INF] Manager daemon y is now available 2024-07-15T06:57:06.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:05 smithi089 bash[31372]: audit 2024-07-15T06:57:04.749337+0000 mon.c (mon.2) 89 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:57:06.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:05 smithi089 bash[31372]: audit 2024-07-15T06:57:04.777579+0000 mon.a (mon.0) 18 : audit [INF] from='mgr.25244 ' entity='mgr.y' 2024-07-15T06:57:06.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:05 smithi089 bash[31372]: audit 2024-07-15T06:57:04.779190+0000 mon.c (mon.2) 90 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:57:06.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:05 smithi089 bash[31372]: audit 2024-07-15T06:57:04.783642+0000 mon.c (mon.2) 91 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-07-15T06:57:06.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:05 smithi089 bash[31372]: audit 2024-07-15T06:57:04.783989+0000 mon.a (mon.0) 19 : audit [INF] from='mgr.25244 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-07-15T06:57:06.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:05 smithi089 bash[31372]: audit 2024-07-15T06:57:04.821432+0000 mon.c (mon.2) 92 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-07-15T06:57:06.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:05 smithi089 bash[31372]: audit 2024-07-15T06:57:04.821897+0000 mon.a (mon.0) 20 : audit [INF] from='mgr.25244 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-07-15T06:57:07.053 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:06 smithi134 bash[24068]: cluster 2024-07-15T06:57:05.722864+0000 mon.a (mon.0) 21 : cluster [DBG] mgrmap e37: y(active, since 1.04734s), standbys: x 2024-07-15T06:57:07.053 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:06 smithi134 bash[24068]: cluster 2024-07-15T06:57:05.738946+0000 mgr.y (mgr.25244) 1 : cluster [DBG] pgmap v3: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:57:07.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:06 smithi089 bash[79810]: cluster 2024-07-15T06:57:05.722864+0000 mon.a (mon.0) 21 : cluster [DBG] mgrmap e37: y(active, since 1.04734s), standbys: x 2024-07-15T06:57:07.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:06 smithi089 bash[79810]: cluster 2024-07-15T06:57:05.738946+0000 mgr.y (mgr.25244) 1 : cluster [DBG] pgmap v3: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:57:07.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:06 smithi089 bash[31372]: cluster 2024-07-15T06:57:05.722864+0000 mon.a (mon.0) 21 : cluster [DBG] mgrmap e37: y(active, since 1.04734s), standbys: x 2024-07-15T06:57:07.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:06 smithi089 bash[31372]: cluster 2024-07-15T06:57:05.738946+0000 mgr.y (mgr.25244) 1 : cluster [DBG] pgmap v3: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:57:07.120 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:57:06 smithi089 bash[72894]: debug 2024-07-15T06:57:06.855+0000 7f10d45fb640 -1 log_channel(cephadm) log [ERR] : [15/Jul/2024:06:57:06] ENGINE Error in HTTPServer.serve 2024-07-15T06:57:07.120 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:57:06 smithi089 bash[72894]: Traceback (most recent call last): 2024-07-15T06:57:07.120 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:57:06 smithi089 bash[72894]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-07-15T06:57:07.120 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:57:06 smithi089 bash[72894]: self._connections.run(self.expiration_interval) 2024-07-15T06:57:07.121 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:57:06 smithi089 bash[72894]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-07-15T06:57:07.121 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:57:06 smithi089 bash[72894]: self._run(expiration_interval) 2024-07-15T06:57:07.121 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:57:06 smithi089 bash[72894]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-07-15T06:57:07.121 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:57:06 smithi089 bash[72894]: new_conn = self._from_server_socket(self.server.socket) 2024-07-15T06:57:07.121 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:57:06 smithi089 bash[72894]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-07-15T06:57:07.121 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:57:06 smithi089 bash[72894]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-07-15T06:57:07.121 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:57:06 smithi089 bash[72894]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-07-15T06:57:07.121 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:57:06 smithi089 bash[72894]: s = self.context.wrap_socket( 2024-07-15T06:57:07.121 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:57:06 smithi089 bash[72894]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-07-15T06:57:07.121 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:57:06 smithi089 bash[72894]: return self.sslsocket_class._create( 2024-07-15T06:57:07.121 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:57:06 smithi089 bash[72894]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-07-15T06:57:07.121 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:57:06 smithi089 bash[72894]: self.do_handshake() 2024-07-15T06:57:07.121 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:57:06 smithi089 bash[72894]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-07-15T06:57:07.121 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:57:06 smithi089 bash[72894]: self._sslobj.do_handshake() 2024-07-15T06:57:07.121 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:57:06 smithi089 bash[72894]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-07-15T06:57:08.053 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:07 smithi134 bash[24068]: cephadm 2024-07-15T06:57:06.629765+0000 mgr.y (mgr.25244) 2 : cephadm [INF] [15/Jul/2024:06:57:06] ENGINE Bus STARTING 2024-07-15T06:57:08.053 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:07 smithi134 bash[24068]: cluster 2024-07-15T06:57:06.698040+0000 mgr.y (mgr.25244) 3 : cluster [DBG] pgmap v4: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:57:08.053 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:07 smithi134 bash[24068]: cephadm 2024-07-15T06:57:06.731732+0000 mgr.y (mgr.25244) 4 : cephadm [INF] [15/Jul/2024:06:57:06] ENGINE Serving on http://172.21.15.89:8765 2024-07-15T06:57:08.053 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:07 smithi134 bash[24068]: cluster 2024-07-15T06:57:06.739911+0000 mon.a (mon.0) 22 : cluster [DBG] mgrmap e38: y(active, since 2s), standbys: x 2024-07-15T06:57:08.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:07 smithi089 bash[79810]: cephadm 2024-07-15T06:57:06.629765+0000 mgr.y (mgr.25244) 2 : cephadm [INF] [15/Jul/2024:06:57:06] ENGINE Bus STARTING 2024-07-15T06:57:08.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:07 smithi089 bash[79810]: cluster 2024-07-15T06:57:06.698040+0000 mgr.y (mgr.25244) 3 : cluster [DBG] pgmap v4: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:57:08.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:07 smithi089 bash[79810]: cephadm 2024-07-15T06:57:06.731732+0000 mgr.y (mgr.25244) 4 : cephadm [INF] [15/Jul/2024:06:57:06] ENGINE Serving on http://172.21.15.89:8765 2024-07-15T06:57:08.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:07 smithi089 bash[79810]: cluster 2024-07-15T06:57:06.739911+0000 mon.a (mon.0) 22 : cluster [DBG] mgrmap e38: y(active, since 2s), standbys: x 2024-07-15T06:57:08.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:07 smithi089 bash[31372]: cephadm 2024-07-15T06:57:06.629765+0000 mgr.y (mgr.25244) 2 : cephadm [INF] [15/Jul/2024:06:57:06] ENGINE Bus STARTING 2024-07-15T06:57:08.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:07 smithi089 bash[31372]: cluster 2024-07-15T06:57:06.698040+0000 mgr.y (mgr.25244) 3 : cluster [DBG] pgmap v4: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:57:08.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:07 smithi089 bash[31372]: cephadm 2024-07-15T06:57:06.731732+0000 mgr.y (mgr.25244) 4 : cephadm [INF] [15/Jul/2024:06:57:06] ENGINE Serving on http://172.21.15.89:8765 2024-07-15T06:57:08.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:07 smithi089 bash[31372]: cluster 2024-07-15T06:57:06.739911+0000 mon.a (mon.0) 22 : cluster [DBG] mgrmap e38: y(active, since 2s), standbys: x 2024-07-15T06:57:08.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:08 smithi089 bash[79810]: cephadm 2024-07-15T06:57:06.856989+0000 mgr.y (mgr.25244) 5 : cephadm [INF] [15/Jul/2024:06:57:06] ENGINE Serving on https://172.21.15.89:7150 2024-07-15T06:57:08.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:08 smithi089 bash[79810]: cephadm 2024-07-15T06:57:06.857217+0000 mgr.y (mgr.25244) 6 : cephadm [INF] [15/Jul/2024:06:57:06] ENGINE Bus STARTED 2024-07-15T06:57:08.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:08 smithi089 bash[79810]: cephadm 2024-07-15T06:57:06.858963+0000 mgr.y (mgr.25244) 7 : cephadm [ERR] [15/Jul/2024:06:57:06] ENGINE Error in HTTPServer.serve 2024-07-15T06:57:08.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:08 smithi089 bash[79810]: Traceback (most recent call last): 2024-07-15T06:57:08.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:08 smithi089 bash[79810]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-07-15T06:57:08.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:08 smithi089 bash[79810]: self._connections.run(self.expiration_interval) 2024-07-15T06:57:08.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:08 smithi089 bash[79810]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-07-15T06:57:08.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:08 smithi089 bash[79810]: self._run(expiration_interval) 2024-07-15T06:57:08.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:08 smithi089 bash[79810]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-07-15T06:57:08.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:08 smithi089 bash[79810]: new_conn = self._from_server_socket(self.server.socket) 2024-07-15T06:57:08.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:08 smithi089 bash[79810]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-07-15T06:57:08.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:08 smithi089 bash[79810]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-07-15T06:57:08.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:08 smithi089 bash[79810]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-07-15T06:57:08.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:08 smithi089 bash[79810]: s = self.context.wrap_socket( 2024-07-15T06:57:08.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:08 smithi089 bash[79810]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-07-15T06:57:08.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:08 smithi089 bash[79810]: return self.sslsocket_class._create( 2024-07-15T06:57:08.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:08 smithi089 bash[79810]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-07-15T06:57:08.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:08 smithi089 bash[79810]: self.do_handshake() 2024-07-15T06:57:08.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:08 smithi089 bash[79810]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-07-15T06:57:08.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:08 smithi089 bash[79810]: self._sslobj.do_handshake() 2024-07-15T06:57:08.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:08 smithi089 bash[79810]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-07-15T06:57:08.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:08 smithi089 bash[31372]: cephadm 2024-07-15T06:57:06.856989+0000 mgr.y (mgr.25244) 5 : cephadm [INF] [15/Jul/2024:06:57:06] ENGINE Serving on https://172.21.15.89:7150 2024-07-15T06:57:08.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:08 smithi089 bash[31372]: cephadm 2024-07-15T06:57:06.857217+0000 mgr.y (mgr.25244) 6 : cephadm [INF] [15/Jul/2024:06:57:06] ENGINE Bus STARTED 2024-07-15T06:57:08.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:08 smithi089 bash[31372]: cephadm 2024-07-15T06:57:06.858963+0000 mgr.y (mgr.25244) 7 : cephadm [ERR] [15/Jul/2024:06:57:06] ENGINE Error in HTTPServer.serve 2024-07-15T06:57:08.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:08 smithi089 bash[31372]: Traceback (most recent call last): 2024-07-15T06:57:08.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:08 smithi089 bash[31372]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-07-15T06:57:08.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:08 smithi089 bash[31372]: self._connections.run(self.expiration_interval) 2024-07-15T06:57:08.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:08 smithi089 bash[31372]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-07-15T06:57:08.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:08 smithi089 bash[31372]: self._run(expiration_interval) 2024-07-15T06:57:08.873 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:08 smithi089 bash[31372]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-07-15T06:57:08.873 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:08 smithi089 bash[31372]: new_conn = self._from_server_socket(self.server.socket) 2024-07-15T06:57:08.873 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:08 smithi089 bash[31372]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-07-15T06:57:08.873 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:08 smithi089 bash[31372]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-07-15T06:57:08.873 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:08 smithi089 bash[31372]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-07-15T06:57:08.873 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:08 smithi089 bash[31372]: s = self.context.wrap_socket( 2024-07-15T06:57:08.873 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:08 smithi089 bash[31372]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-07-15T06:57:08.873 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:08 smithi089 bash[31372]: return self.sslsocket_class._create( 2024-07-15T06:57:08.873 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:08 smithi089 bash[31372]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-07-15T06:57:08.873 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:08 smithi089 bash[31372]: self.do_handshake() 2024-07-15T06:57:08.873 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:08 smithi089 bash[31372]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-07-15T06:57:08.873 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:08 smithi089 bash[31372]: self._sslobj.do_handshake() 2024-07-15T06:57:08.873 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:08 smithi089 bash[31372]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-07-15T06:57:08.889 INFO:teuthology.orchestra.run.smithi089.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-07-15T06:57:08.889 INFO:teuthology.orchestra.run.smithi089.stdout:alertmanager.a smithi089 *:9093,9094 running (102s) 59s ago 9m 13.8M - 0.25.0 c8568f914cd2 c476af8941dd 2024-07-15T06:57:08.890 INFO:teuthology.orchestra.run.smithi089.stdout:grafana.a smithi134 *:3000 running (89s) 29s ago 8m 73.8M - 9.4.7 954c08fa6188 ac957811e4af 2024-07-15T06:57:08.890 INFO:teuthology.orchestra.run.smithi089.stdout:mds.foo.smithi089.cczuoz smithi089 running (6m) 59s ago 6m 13.2M - 17.2.7-1102-g9a202024 280b03ed6869 a1f028849336 2024-07-15T06:57:08.890 INFO:teuthology.orchestra.run.smithi089.stdout:mds.foo.smithi134.wugtzh smithi134 running (6m) 29s ago 6m 15.0M - 17.2.7-1102-g9a202024 280b03ed6869 fb5bbeafa32b 2024-07-15T06:57:08.890 INFO:teuthology.orchestra.run.smithi089.stdout:mgr.x smithi134 *:8443,9283,8765 running (36s) 29s ago 17m 173M - 18.2.2-1609-g6ea7aaa9 0add76435ed9 842a5eaedc06 2024-07-15T06:57:08.890 INFO:teuthology.orchestra.run.smithi089.stdout:mgr.y smithi089 *:8443,9283,8765 running (83s) 59s ago 19m 455M - 18.2.2-1609-g6ea7aaa9 0add76435ed9 20c56e260e99 2024-07-15T06:57:08.890 INFO:teuthology.orchestra.run.smithi089.stdout:mon.a smithi089 running (19m) 59s ago 20m 54.3M 2048M 17.2.7-1102-g9a202024 280b03ed6869 40a92d660143 2024-07-15T06:57:08.890 INFO:teuthology.orchestra.run.smithi089.stdout:mon.b smithi134 running (17m) 29s ago 17m 43.6M 2048M 17.2.7-1102-g9a202024 280b03ed6869 19182ed25bde 2024-07-15T06:57:08.890 INFO:teuthology.orchestra.run.smithi089.stdout:mon.c smithi089 running (17m) 59s ago 17m 42.6M 2048M 17.2.7-1102-g9a202024 280b03ed6869 ad83089d3cbf 2024-07-15T06:57:08.890 INFO:teuthology.orchestra.run.smithi089.stdout:node-exporter.a smithi089 *:9100 running (108s) 59s ago 9m 7455k - 1.5.0 0da6a335fe13 5da5d79c182f 2024-07-15T06:57:08.890 INFO:teuthology.orchestra.run.smithi089.stdout:node-exporter.b smithi134 *:9100 running (95s) 29s ago 9m 7671k - 1.5.0 0da6a335fe13 9ace5df7006d 2024-07-15T06:57:08.890 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0 smithi089 running (16m) 59s ago 16m 49.3M 2524M 17.2.7-1102-g9a202024 280b03ed6869 3fd6dc329fe6 2024-07-15T06:57:08.890 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1 smithi089 running (15m) 59s ago 15m 49.0M 2524M 17.2.7-1102-g9a202024 280b03ed6869 b4ff988d2ed5 2024-07-15T06:57:08.890 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2 smithi089 running (14m) 59s ago 14m 55.9M 2524M 17.2.7-1102-g9a202024 280b03ed6869 428fbfd265d1 2024-07-15T06:57:08.891 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3 smithi089 running (14m) 59s ago 14m 46.4M 2524M 17.2.7-1102-g9a202024 280b03ed6869 4f43743cd007 2024-07-15T06:57:08.891 INFO:teuthology.orchestra.run.smithi089.stdout:osd.4 smithi134 running (13m) 29s ago 13m 50.4M 2524M 17.2.7-1102-g9a202024 280b03ed6869 2ae2fa51d983 2024-07-15T06:57:08.891 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5 smithi134 running (12m) 29s ago 12m 54.2M 2524M 17.2.7-1102-g9a202024 280b03ed6869 61be14c58fc3 2024-07-15T06:57:08.891 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6 smithi134 running (11m) 29s ago 11m 52.8M 2524M 17.2.7-1102-g9a202024 280b03ed6869 f92c7dd84666 2024-07-15T06:57:08.891 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7 smithi134 running (10m) 29s ago 10m 53.0M 2524M 17.2.7-1102-g9a202024 280b03ed6869 e44d11319ff6 2024-07-15T06:57:08.891 INFO:teuthology.orchestra.run.smithi089.stdout:prometheus.a smithi134 *:9095 running (43s) 29s ago 9m 41.0M - 2.43.0 a07b618ecd1d 8873e527bacf 2024-07-15T06:57:09.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:08 smithi134 bash[24068]: cephadm 2024-07-15T06:57:06.856989+0000 mgr.y (mgr.25244) 5 : cephadm [INF] [15/Jul/2024:06:57:06] ENGINE Serving on https://172.21.15.89:7150 2024-07-15T06:57:09.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:08 smithi134 bash[24068]: cephadm 2024-07-15T06:57:06.857217+0000 mgr.y (mgr.25244) 6 : cephadm [INF] [15/Jul/2024:06:57:06] ENGINE Bus STARTED 2024-07-15T06:57:09.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:08 smithi134 bash[24068]: cephadm 2024-07-15T06:57:06.858963+0000 mgr.y (mgr.25244) 7 : cephadm [ERR] [15/Jul/2024:06:57:06] ENGINE Error in HTTPServer.serve 2024-07-15T06:57:09.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:08 smithi134 bash[24068]: Traceback (most recent call last): 2024-07-15T06:57:09.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:08 smithi134 bash[24068]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-07-15T06:57:09.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:08 smithi134 bash[24068]: self._connections.run(self.expiration_interval) 2024-07-15T06:57:09.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:08 smithi134 bash[24068]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-07-15T06:57:09.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:08 smithi134 bash[24068]: self._run(expiration_interval) 2024-07-15T06:57:09.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:08 smithi134 bash[24068]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-07-15T06:57:09.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:08 smithi134 bash[24068]: new_conn = self._from_server_socket(self.server.socket) 2024-07-15T06:57:09.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:08 smithi134 bash[24068]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-07-15T06:57:09.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:08 smithi134 bash[24068]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-07-15T06:57:09.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:08 smithi134 bash[24068]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-07-15T06:57:09.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:08 smithi134 bash[24068]: s = self.context.wrap_socket( 2024-07-15T06:57:09.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:08 smithi134 bash[24068]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-07-15T06:57:09.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:08 smithi134 bash[24068]: return self.sslsocket_class._create( 2024-07-15T06:57:09.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:08 smithi134 bash[24068]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-07-15T06:57:09.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:08 smithi134 bash[24068]: self.do_handshake() 2024-07-15T06:57:09.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:08 smithi134 bash[24068]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-07-15T06:57:09.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:08 smithi134 bash[24068]: self._sslobj.do_handshake() 2024-07-15T06:57:09.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:08 smithi134 bash[24068]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-07-15T06:57:09.652 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -e sha1=6ea7aaa9f93d34dc4d21862afe00140dfddbd706 -- bash -c 'ceph orch upgrade pause' 2024-07-15T06:57:10.053 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:09 smithi134 bash[24068]: cluster 2024-07-15T06:57:08.698406+0000 mgr.y (mgr.25244) 8 : cluster [DBG] pgmap v5: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:57:10.053 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:09 smithi134 bash[24068]: cluster 2024-07-15T06:57:08.751246+0000 mon.a (mon.0) 23 : cluster [DBG] mgrmap e39: y(active, since 4s), standbys: x 2024-07-15T06:57:10.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:09 smithi089 bash[31372]: cluster 2024-07-15T06:57:08.698406+0000 mgr.y (mgr.25244) 8 : cluster [DBG] pgmap v5: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:57:10.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:09 smithi089 bash[31372]: cluster 2024-07-15T06:57:08.751246+0000 mon.a (mon.0) 23 : cluster [DBG] mgrmap e39: y(active, since 4s), standbys: x 2024-07-15T06:57:10.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:09 smithi089 bash[79810]: cluster 2024-07-15T06:57:08.698406+0000 mgr.y (mgr.25244) 8 : cluster [DBG] pgmap v5: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:57:10.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:09 smithi089 bash[79810]: cluster 2024-07-15T06:57:08.751246+0000 mon.a (mon.0) 23 : cluster [DBG] mgrmap e39: y(active, since 4s), standbys: x 2024-07-15T06:57:11.053 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:10 smithi134 bash[24068]: audit 2024-07-15T06:57:08.871910+0000 mgr.y (mgr.25244) 9 : audit [DBG] from='client.34149 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T06:57:11.053 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:10 smithi134 bash[24068]: audit 2024-07-15T06:57:10.732434+0000 mon.a (mon.0) 24 : audit [INF] from='mgr.25244 ' entity='mgr.y' 2024-07-15T06:57:11.053 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:10 smithi134 bash[24068]: audit 2024-07-15T06:57:10.738204+0000 mon.a (mon.0) 25 : audit [INF] from='mgr.25244 ' entity='mgr.y' 2024-07-15T06:57:11.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:10 smithi089 bash[79810]: audit 2024-07-15T06:57:08.871910+0000 mgr.y (mgr.25244) 9 : audit [DBG] from='client.34149 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T06:57:11.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:10 smithi089 bash[79810]: audit 2024-07-15T06:57:10.732434+0000 mon.a (mon.0) 24 : audit [INF] from='mgr.25244 ' entity='mgr.y' 2024-07-15T06:57:11.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:10 smithi089 bash[79810]: audit 2024-07-15T06:57:10.738204+0000 mon.a (mon.0) 25 : audit [INF] from='mgr.25244 ' entity='mgr.y' 2024-07-15T06:57:11.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:10 smithi089 bash[31372]: audit 2024-07-15T06:57:08.871910+0000 mgr.y (mgr.25244) 9 : audit [DBG] from='client.34149 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T06:57:11.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:10 smithi089 bash[31372]: audit 2024-07-15T06:57:10.732434+0000 mon.a (mon.0) 24 : audit [INF] from='mgr.25244 ' entity='mgr.y' 2024-07-15T06:57:11.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:10 smithi089 bash[31372]: audit 2024-07-15T06:57:10.738204+0000 mon.a (mon.0) 25 : audit [INF] from='mgr.25244 ' entity='mgr.y' 2024-07-15T06:57:12.053 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:11 smithi134 bash[24068]: cluster 2024-07-15T06:57:10.698750+0000 mgr.y (mgr.25244) 10 : cluster [DBG] pgmap v6: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:57:12.053 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:11 smithi134 bash[24068]: audit 2024-07-15T06:57:11.026328+0000 mon.a (mon.0) 26 : audit [INF] from='mgr.25244 ' entity='mgr.y' 2024-07-15T06:57:12.053 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:11 smithi134 bash[24068]: audit 2024-07-15T06:57:11.036297+0000 mon.a (mon.0) 27 : audit [INF] from='mgr.25244 ' entity='mgr.y' 2024-07-15T06:57:12.053 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:11 smithi134 bash[24068]: audit 2024-07-15T06:57:11.460017+0000 mon.a (mon.0) 28 : audit [INF] from='mgr.25244 ' entity='mgr.y' 2024-07-15T06:57:12.053 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:11 smithi134 bash[24068]: audit 2024-07-15T06:57:11.473416+0000 mon.a (mon.0) 29 : audit [INF] from='mgr.25244 ' entity='mgr.y' 2024-07-15T06:57:12.053 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:11 smithi134 bash[24068]: audit 2024-07-15T06:57:11.730146+0000 mon.a (mon.0) 30 : audit [INF] from='mgr.25244 ' entity='mgr.y' 2024-07-15T06:57:12.053 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:11 smithi134 bash[24068]: audit 2024-07-15T06:57:11.744848+0000 mon.a (mon.0) 31 : audit [INF] from='mgr.25244 ' entity='mgr.y' 2024-07-15T06:57:12.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:11 smithi089 bash[79810]: cluster 2024-07-15T06:57:10.698750+0000 mgr.y (mgr.25244) 10 : cluster [DBG] pgmap v6: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:57:12.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:11 smithi089 bash[79810]: audit 2024-07-15T06:57:11.026328+0000 mon.a (mon.0) 26 : audit [INF] from='mgr.25244 ' entity='mgr.y' 2024-07-15T06:57:12.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:11 smithi089 bash[79810]: audit 2024-07-15T06:57:11.036297+0000 mon.a (mon.0) 27 : audit [INF] from='mgr.25244 ' entity='mgr.y' 2024-07-15T06:57:12.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:11 smithi089 bash[79810]: audit 2024-07-15T06:57:11.460017+0000 mon.a (mon.0) 28 : audit [INF] from='mgr.25244 ' entity='mgr.y' 2024-07-15T06:57:12.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:11 smithi089 bash[79810]: audit 2024-07-15T06:57:11.473416+0000 mon.a (mon.0) 29 : audit [INF] from='mgr.25244 ' entity='mgr.y' 2024-07-15T06:57:12.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:11 smithi089 bash[79810]: audit 2024-07-15T06:57:11.730146+0000 mon.a (mon.0) 30 : audit [INF] from='mgr.25244 ' entity='mgr.y' 2024-07-15T06:57:12.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:11 smithi089 bash[79810]: audit 2024-07-15T06:57:11.744848+0000 mon.a (mon.0) 31 : audit [INF] from='mgr.25244 ' entity='mgr.y' 2024-07-15T06:57:12.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:11 smithi089 bash[31372]: cluster 2024-07-15T06:57:10.698750+0000 mgr.y (mgr.25244) 10 : cluster [DBG] pgmap v6: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:57:12.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:11 smithi089 bash[31372]: audit 2024-07-15T06:57:11.026328+0000 mon.a (mon.0) 26 : audit [INF] from='mgr.25244 ' entity='mgr.y' 2024-07-15T06:57:12.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:11 smithi089 bash[31372]: audit 2024-07-15T06:57:11.036297+0000 mon.a (mon.0) 27 : audit [INF] from='mgr.25244 ' entity='mgr.y' 2024-07-15T06:57:12.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:11 smithi089 bash[31372]: audit 2024-07-15T06:57:11.460017+0000 mon.a (mon.0) 28 : audit [INF] from='mgr.25244 ' entity='mgr.y' 2024-07-15T06:57:12.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:11 smithi089 bash[31372]: audit 2024-07-15T06:57:11.473416+0000 mon.a (mon.0) 29 : audit [INF] from='mgr.25244 ' entity='mgr.y' 2024-07-15T06:57:12.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:11 smithi089 bash[31372]: audit 2024-07-15T06:57:11.730146+0000 mon.a (mon.0) 30 : audit [INF] from='mgr.25244 ' entity='mgr.y' 2024-07-15T06:57:12.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:11 smithi089 bash[31372]: audit 2024-07-15T06:57:11.744848+0000 mon.a (mon.0) 31 : audit [INF] from='mgr.25244 ' entity='mgr.y' 2024-07-15T06:57:12.803 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:57:12 smithi134 bash[60729]: ts=2024-07-15T06:57:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:57:13.502 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:13 smithi134 bash[24068]: cluster 2024-07-15T06:57:12.699851+0000 mgr.y (mgr.25244) 11 : cluster [DBG] pgmap v7: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 16 op/s 2024-07-15T06:57:13.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:13 smithi089 bash[79810]: cluster 2024-07-15T06:57:12.699851+0000 mgr.y (mgr.25244) 11 : cluster [DBG] pgmap v7: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 16 op/s 2024-07-15T06:57:13.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:13 smithi089 bash[31372]: cluster 2024-07-15T06:57:12.699851+0000 mgr.y (mgr.25244) 11 : cluster [DBG] pgmap v7: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 16 op/s 2024-07-15T06:57:13.803 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:57:13 smithi134 bash[60729]: ts=2024-07-15T06:57:13.508Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi134\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi134\", job=\"node\", machine=\"x86_64\", nodename=\"smithi134\", release=\"5.15.0-113-generic\", sysname=\"Linux\", version=\"#123-Ubuntu SMP Mon Jun 10 08:16:17 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", domainname=\"(none)\", instance=\"smithi134\", job=\"node\", machine=\"x86_64\", nodename=\"smithi134\", release=\"5.15.0-113-generic\", sysname=\"Linux\", version=\"#123-Ubuntu SMP Mon Jun 10 08:16:17 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:57:14.182 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T06:57:14.619 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:57:14 smithi089 bash[72894]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:57:14] "GET /metrics HTTP/1.1" 200 31229 "" "Prometheus/2.43.0" 2024-07-15T06:57:14.803 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:57:14 smithi134 bash[60729]: ts=2024-07-15T06:57:14.454Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:57:14.803 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:57:14 smithi134 bash[60729]: ts=2024-07-15T06:57:14.455Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:57:16.053 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:15 smithi134 bash[24068]: cluster 2024-07-15T06:57:14.700678+0000 mgr.y (mgr.25244) 12 : cluster [DBG] pgmap v8: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-07-15T06:57:16.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:15 smithi089 bash[79810]: cluster 2024-07-15T06:57:14.700678+0000 mgr.y (mgr.25244) 12 : cluster [DBG] pgmap v8: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-07-15T06:57:16.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:15 smithi089 bash[31372]: cluster 2024-07-15T06:57:14.700678+0000 mgr.y (mgr.25244) 12 : cluster [DBG] pgmap v8: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-07-15T06:57:17.869 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:17 smithi089 bash[79810]: cluster 2024-07-15T06:57:16.701768+0000 mgr.y (mgr.25244) 13 : cluster [DBG] pgmap v9: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-07-15T06:57:17.870 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:17 smithi089 bash[31372]: cluster 2024-07-15T06:57:16.701768+0000 mgr.y (mgr.25244) 13 : cluster [DBG] pgmap v9: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-07-15T06:57:17.906 INFO:teuthology.orchestra.run.smithi089.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:badc66b92d2a2cfbbce7f293b5accdc50f486d2b2d966876e46fe28d05f464f1 2024-07-15T06:57:17.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:17 smithi134 bash[24068]: cluster 2024-07-15T06:57:16.701768+0000 mgr.y (mgr.25244) 13 : cluster [DBG] pgmap v9: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-07-15T06:57:18.732 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -e sha1=6ea7aaa9f93d34dc4d21862afe00140dfddbd706 -- bash -c 'sleep 60' 2024-07-15T06:57:18.905 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:57:18 smithi134 bash[60729]: ts=2024-07-15T06:57:18.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1102-g9a202024 (9a2020246f85463d4fdb115c94aa40b463caa1d2) quincy (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1102-g9a202024 (9a2020246f85463d4fdb115c94aa40b463caa1d2) quincy (stable)\", cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:57:19.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:18 smithi089 bash[79810]: audit 2024-07-15T06:57:17.902721+0000 mgr.y (mgr.25244) 14 : audit [DBG] from='client.34155 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T06:57:19.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:18 smithi089 bash[79810]: cephadm 2024-07-15T06:57:17.903479+0000 mgr.y (mgr.25244) 15 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:badc66b92d2a2cfbbce7f293b5accdc50f486d2b2d966876e46fe28d05f464f1 2024-07-15T06:57:19.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:18 smithi089 bash[79810]: audit 2024-07-15T06:57:17.909236+0000 mon.a (mon.0) 32 : audit [INF] from='mgr.25244 ' entity='mgr.y' 2024-07-15T06:57:19.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:18 smithi089 bash[79810]: cluster 2024-07-15T06:57:18.702446+0000 mgr.y (mgr.25244) 16 : cluster [DBG] pgmap v10: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-07-15T06:57:19.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:18 smithi089 bash[31372]: audit 2024-07-15T06:57:17.902721+0000 mgr.y (mgr.25244) 14 : audit [DBG] from='client.34155 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T06:57:19.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:18 smithi089 bash[31372]: cephadm 2024-07-15T06:57:17.903479+0000 mgr.y (mgr.25244) 15 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:badc66b92d2a2cfbbce7f293b5accdc50f486d2b2d966876e46fe28d05f464f1 2024-07-15T06:57:19.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:18 smithi089 bash[31372]: audit 2024-07-15T06:57:17.909236+0000 mon.a (mon.0) 32 : audit [INF] from='mgr.25244 ' entity='mgr.y' 2024-07-15T06:57:19.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:18 smithi089 bash[31372]: cluster 2024-07-15T06:57:18.702446+0000 mgr.y (mgr.25244) 16 : cluster [DBG] pgmap v10: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-07-15T06:57:19.303 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:18 smithi134 bash[24068]: audit 2024-07-15T06:57:17.902721+0000 mgr.y (mgr.25244) 14 : audit [DBG] from='client.34155 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T06:57:19.303 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:18 smithi134 bash[24068]: cephadm 2024-07-15T06:57:17.903479+0000 mgr.y (mgr.25244) 15 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:badc66b92d2a2cfbbce7f293b5accdc50f486d2b2d966876e46fe28d05f464f1 2024-07-15T06:57:19.303 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:18 smithi134 bash[24068]: audit 2024-07-15T06:57:17.909236+0000 mon.a (mon.0) 32 : audit [INF] from='mgr.25244 ' entity='mgr.y' 2024-07-15T06:57:19.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:18 smithi134 bash[24068]: cluster 2024-07-15T06:57:18.702446+0000 mgr.y (mgr.25244) 16 : cluster [DBG] pgmap v10: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-07-15T06:57:20.053 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:19 smithi134 bash[24068]: audit 2024-07-15T06:57:19.772070+0000 mon.c (mon.2) 93 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:57:20.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:19 smithi089 bash[79810]: audit 2024-07-15T06:57:19.772070+0000 mon.c (mon.2) 93 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:57:20.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:19 smithi089 bash[31372]: audit 2024-07-15T06:57:19.772070+0000 mon.c (mon.2) 93 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:57:21.053 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:20 smithi134 bash[24068]: cluster 2024-07-15T06:57:20.703148+0000 mgr.y (mgr.25244) 17 : cluster [DBG] pgmap v11: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-07-15T06:57:21.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:20 smithi089 bash[79810]: cluster 2024-07-15T06:57:20.703148+0000 mgr.y (mgr.25244) 17 : cluster [DBG] pgmap v11: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-07-15T06:57:21.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:20 smithi089 bash[31372]: cluster 2024-07-15T06:57:20.703148+0000 mgr.y (mgr.25244) 17 : cluster [DBG] pgmap v11: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-07-15T06:57:22.803 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:57:22 smithi134 bash[60729]: ts=2024-07-15T06:57:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:57:22.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:22 smithi134 bash[24068]: audit 2024-07-15T06:57:21.524994+0000 mon.a (mon.0) 33 : audit [INF] from='mgr.25244 ' entity='mgr.y' 2024-07-15T06:57:22.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:22 smithi134 bash[24068]: audit 2024-07-15T06:57:21.533196+0000 mon.a (mon.0) 34 : audit [INF] from='mgr.25244 ' entity='mgr.y' 2024-07-15T06:57:22.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:22 smithi134 bash[24068]: audit 2024-07-15T06:57:21.536356+0000 mon.c (mon.2) 94 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:57:22.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:22 smithi134 bash[24068]: audit 2024-07-15T06:57:21.537335+0000 mon.c (mon.2) 95 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:57:22.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:22 smithi134 bash[24068]: cephadm 2024-07-15T06:57:21.538609+0000 mgr.y (mgr.25244) 18 : cephadm [INF] Updating smithi089:/etc/ceph/ceph.conf 2024-07-15T06:57:22.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:22 smithi134 bash[24068]: cephadm 2024-07-15T06:57:21.538838+0000 mgr.y (mgr.25244) 19 : cephadm [INF] Updating smithi134:/etc/ceph/ceph.conf 2024-07-15T06:57:22.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:22 smithi134 bash[24068]: cephadm 2024-07-15T06:57:21.660919+0000 mgr.y (mgr.25244) 20 : cephadm [INF] Updating smithi089:/var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/config/ceph.conf 2024-07-15T06:57:22.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:22 smithi134 bash[24068]: cephadm 2024-07-15T06:57:21.683504+0000 mgr.y (mgr.25244) 21 : cephadm [INF] Updating smithi134:/var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/config/ceph.conf 2024-07-15T06:57:22.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:22 smithi134 bash[24068]: cephadm 2024-07-15T06:57:21.782508+0000 mgr.y (mgr.25244) 22 : cephadm [INF] Updating smithi089:/etc/ceph/ceph.client.admin.keyring 2024-07-15T06:57:22.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:22 smithi134 bash[24068]: audit 2024-07-15T06:57:22.029699+0000 mon.a (mon.0) 35 : audit [INF] from='mgr.25244 ' entity='mgr.y' 2024-07-15T06:57:22.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:22 smithi134 bash[24068]: audit 2024-07-15T06:57:22.037545+0000 mon.a (mon.0) 36 : audit [INF] from='mgr.25244 ' entity='mgr.y' 2024-07-15T06:57:22.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:22 smithi134 bash[24068]: audit 2024-07-15T06:57:22.086484+0000 mon.a (mon.0) 37 : audit [INF] from='mgr.25244 ' entity='mgr.y' 2024-07-15T06:57:22.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:22 smithi134 bash[24068]: audit 2024-07-15T06:57:22.099690+0000 mon.a (mon.0) 38 : audit [INF] from='mgr.25244 ' entity='mgr.y' 2024-07-15T06:57:22.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:22 smithi134 bash[24068]: audit 2024-07-15T06:57:22.110889+0000 mon.a (mon.0) 39 : audit [INF] from='mgr.25244 ' entity='mgr.y' 2024-07-15T06:57:22.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:22 smithi134 bash[24068]: audit 2024-07-15T06:57:22.236089+0000 mon.c (mon.2) 96 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:57:22.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:22 smithi134 bash[24068]: audit 2024-07-15T06:57:22.238511+0000 mon.c (mon.2) 97 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:57:22.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:22 smithi134 bash[24068]: audit 2024-07-15T06:57:22.239681+0000 mon.c (mon.2) 98 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:57:22.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:22 smithi134 bash[24068]: audit 2024-07-15T06:57:22.249697+0000 mon.a (mon.0) 40 : audit [INF] from='mgr.25244 ' entity='mgr.y' 2024-07-15T06:57:22.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:22 smithi089 bash[79810]: audit 2024-07-15T06:57:21.524994+0000 mon.a (mon.0) 33 : audit [INF] from='mgr.25244 ' entity='mgr.y' 2024-07-15T06:57:22.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:22 smithi089 bash[79810]: audit 2024-07-15T06:57:21.533196+0000 mon.a (mon.0) 34 : audit [INF] from='mgr.25244 ' entity='mgr.y' 2024-07-15T06:57:22.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:22 smithi089 bash[79810]: audit 2024-07-15T06:57:21.536356+0000 mon.c (mon.2) 94 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:57:22.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:22 smithi089 bash[79810]: audit 2024-07-15T06:57:21.537335+0000 mon.c (mon.2) 95 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:57:22.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:22 smithi089 bash[79810]: cephadm 2024-07-15T06:57:21.538609+0000 mgr.y (mgr.25244) 18 : cephadm [INF] Updating smithi089:/etc/ceph/ceph.conf 2024-07-15T06:57:22.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:22 smithi089 bash[79810]: cephadm 2024-07-15T06:57:21.538838+0000 mgr.y (mgr.25244) 19 : cephadm [INF] Updating smithi134:/etc/ceph/ceph.conf 2024-07-15T06:57:22.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:22 smithi089 bash[79810]: cephadm 2024-07-15T06:57:21.660919+0000 mgr.y (mgr.25244) 20 : cephadm [INF] Updating smithi089:/var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/config/ceph.conf 2024-07-15T06:57:22.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:22 smithi089 bash[79810]: cephadm 2024-07-15T06:57:21.683504+0000 mgr.y (mgr.25244) 21 : cephadm [INF] Updating smithi134:/var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/config/ceph.conf 2024-07-15T06:57:22.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:22 smithi089 bash[79810]: cephadm 2024-07-15T06:57:21.782508+0000 mgr.y (mgr.25244) 22 : cephadm [INF] Updating smithi089:/etc/ceph/ceph.client.admin.keyring 2024-07-15T06:57:22.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:22 smithi089 bash[79810]: audit 2024-07-15T06:57:22.029699+0000 mon.a (mon.0) 35 : audit [INF] from='mgr.25244 ' entity='mgr.y' 2024-07-15T06:57:22.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:22 smithi089 bash[79810]: audit 2024-07-15T06:57:22.037545+0000 mon.a (mon.0) 36 : audit [INF] from='mgr.25244 ' entity='mgr.y' 2024-07-15T06:57:22.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:22 smithi089 bash[79810]: audit 2024-07-15T06:57:22.086484+0000 mon.a (mon.0) 37 : audit [INF] from='mgr.25244 ' entity='mgr.y' 2024-07-15T06:57:22.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:22 smithi089 bash[79810]: audit 2024-07-15T06:57:22.099690+0000 mon.a (mon.0) 38 : audit [INF] from='mgr.25244 ' entity='mgr.y' 2024-07-15T06:57:22.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:22 smithi089 bash[79810]: audit 2024-07-15T06:57:22.110889+0000 mon.a (mon.0) 39 : audit [INF] from='mgr.25244 ' entity='mgr.y' 2024-07-15T06:57:22.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:22 smithi089 bash[79810]: audit 2024-07-15T06:57:22.236089+0000 mon.c (mon.2) 96 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:57:22.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:22 smithi089 bash[79810]: audit 2024-07-15T06:57:22.238511+0000 mon.c (mon.2) 97 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:57:22.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:22 smithi089 bash[79810]: audit 2024-07-15T06:57:22.239681+0000 mon.c (mon.2) 98 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:57:22.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:22 smithi089 bash[79810]: audit 2024-07-15T06:57:22.249697+0000 mon.a (mon.0) 40 : audit [INF] from='mgr.25244 ' entity='mgr.y' 2024-07-15T06:57:22.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:22 smithi089 bash[31372]: audit 2024-07-15T06:57:21.524994+0000 mon.a (mon.0) 33 : audit [INF] from='mgr.25244 ' entity='mgr.y' 2024-07-15T06:57:22.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:22 smithi089 bash[31372]: audit 2024-07-15T06:57:21.533196+0000 mon.a (mon.0) 34 : audit [INF] from='mgr.25244 ' entity='mgr.y' 2024-07-15T06:57:22.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:22 smithi089 bash[31372]: audit 2024-07-15T06:57:21.536356+0000 mon.c (mon.2) 94 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:57:22.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:22 smithi089 bash[31372]: audit 2024-07-15T06:57:21.537335+0000 mon.c (mon.2) 95 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:57:22.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:22 smithi089 bash[31372]: cephadm 2024-07-15T06:57:21.538609+0000 mgr.y (mgr.25244) 18 : cephadm [INF] Updating smithi089:/etc/ceph/ceph.conf 2024-07-15T06:57:22.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:22 smithi089 bash[31372]: cephadm 2024-07-15T06:57:21.538838+0000 mgr.y (mgr.25244) 19 : cephadm [INF] Updating smithi134:/etc/ceph/ceph.conf 2024-07-15T06:57:22.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:22 smithi089 bash[31372]: cephadm 2024-07-15T06:57:21.660919+0000 mgr.y (mgr.25244) 20 : cephadm [INF] Updating smithi089:/var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/config/ceph.conf 2024-07-15T06:57:22.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:22 smithi089 bash[31372]: cephadm 2024-07-15T06:57:21.683504+0000 mgr.y (mgr.25244) 21 : cephadm [INF] Updating smithi134:/var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/config/ceph.conf 2024-07-15T06:57:22.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:22 smithi089 bash[31372]: cephadm 2024-07-15T06:57:21.782508+0000 mgr.y (mgr.25244) 22 : cephadm [INF] Updating smithi089:/etc/ceph/ceph.client.admin.keyring 2024-07-15T06:57:22.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:22 smithi089 bash[31372]: audit 2024-07-15T06:57:22.029699+0000 mon.a (mon.0) 35 : audit [INF] from='mgr.25244 ' entity='mgr.y' 2024-07-15T06:57:22.873 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:22 smithi089 bash[31372]: audit 2024-07-15T06:57:22.037545+0000 mon.a (mon.0) 36 : audit [INF] from='mgr.25244 ' entity='mgr.y' 2024-07-15T06:57:22.873 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:22 smithi089 bash[31372]: audit 2024-07-15T06:57:22.086484+0000 mon.a (mon.0) 37 : audit [INF] from='mgr.25244 ' entity='mgr.y' 2024-07-15T06:57:22.873 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:22 smithi089 bash[31372]: audit 2024-07-15T06:57:22.099690+0000 mon.a (mon.0) 38 : audit [INF] from='mgr.25244 ' entity='mgr.y' 2024-07-15T06:57:22.873 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:22 smithi089 bash[31372]: audit 2024-07-15T06:57:22.110889+0000 mon.a (mon.0) 39 : audit [INF] from='mgr.25244 ' entity='mgr.y' 2024-07-15T06:57:22.873 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:22 smithi089 bash[31372]: audit 2024-07-15T06:57:22.236089+0000 mon.c (mon.2) 96 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:57:22.873 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:22 smithi089 bash[31372]: audit 2024-07-15T06:57:22.238511+0000 mon.c (mon.2) 97 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:57:22.873 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:22 smithi089 bash[31372]: audit 2024-07-15T06:57:22.239681+0000 mon.c (mon.2) 98 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:57:22.873 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:22 smithi089 bash[31372]: audit 2024-07-15T06:57:22.249697+0000 mon.a (mon.0) 40 : audit [INF] from='mgr.25244 ' entity='mgr.y' 2024-07-15T06:57:23.498 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T06:57:23.803 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:57:23 smithi134 bash[60729]: ts=2024-07-15T06:57:23.508Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi134\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi134\", job=\"node\", machine=\"x86_64\", nodename=\"smithi134\", release=\"5.15.0-113-generic\", sysname=\"Linux\", version=\"#123-Ubuntu SMP Mon Jun 10 08:16:17 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", domainname=\"(none)\", instance=\"smithi134\", job=\"node\", machine=\"x86_64\", nodename=\"smithi134\", release=\"5.15.0-113-generic\", sysname=\"Linux\", version=\"#123-Ubuntu SMP Mon Jun 10 08:16:17 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:57:23.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:23 smithi134 bash[24068]: cephadm 2024-07-15T06:57:21.821438+0000 mgr.y (mgr.25244) 23 : cephadm [INF] Updating smithi134:/etc/ceph/ceph.client.admin.keyring 2024-07-15T06:57:23.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:23 smithi134 bash[24068]: cephadm 2024-07-15T06:57:21.907104+0000 mgr.y (mgr.25244) 24 : cephadm [INF] Updating smithi089:/var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/config/ceph.client.admin.keyring 2024-07-15T06:57:23.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:23 smithi134 bash[24068]: cephadm 2024-07-15T06:57:21.955673+0000 mgr.y (mgr.25244) 25 : cephadm [INF] Updating smithi134:/var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/config/ceph.client.admin.keyring 2024-07-15T06:57:23.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:23 smithi134 bash[24068]: cluster 2024-07-15T06:57:22.704192+0000 mgr.y (mgr.25244) 26 : cluster [DBG] pgmap v12: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-07-15T06:57:23.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:23 smithi089 bash[79810]: cephadm 2024-07-15T06:57:21.821438+0000 mgr.y (mgr.25244) 23 : cephadm [INF] Updating smithi134:/etc/ceph/ceph.client.admin.keyring 2024-07-15T06:57:23.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:23 smithi089 bash[79810]: cephadm 2024-07-15T06:57:21.907104+0000 mgr.y (mgr.25244) 24 : cephadm [INF] Updating smithi089:/var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/config/ceph.client.admin.keyring 2024-07-15T06:57:23.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:23 smithi089 bash[79810]: cephadm 2024-07-15T06:57:21.955673+0000 mgr.y (mgr.25244) 25 : cephadm [INF] Updating smithi134:/var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/config/ceph.client.admin.keyring 2024-07-15T06:57:23.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:23 smithi089 bash[79810]: cluster 2024-07-15T06:57:22.704192+0000 mgr.y (mgr.25244) 26 : cluster [DBG] pgmap v12: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-07-15T06:57:23.870 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:23 smithi089 bash[31372]: cephadm 2024-07-15T06:57:21.821438+0000 mgr.y (mgr.25244) 23 : cephadm [INF] Updating smithi134:/etc/ceph/ceph.client.admin.keyring 2024-07-15T06:57:23.870 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:23 smithi089 bash[31372]: cephadm 2024-07-15T06:57:21.907104+0000 mgr.y (mgr.25244) 24 : cephadm [INF] Updating smithi089:/var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/config/ceph.client.admin.keyring 2024-07-15T06:57:23.870 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:23 smithi089 bash[31372]: cephadm 2024-07-15T06:57:21.955673+0000 mgr.y (mgr.25244) 25 : cephadm [INF] Updating smithi134:/var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/config/ceph.client.admin.keyring 2024-07-15T06:57:23.870 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:23 smithi089 bash[31372]: cluster 2024-07-15T06:57:22.704192+0000 mgr.y (mgr.25244) 26 : cluster [DBG] pgmap v12: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-07-15T06:57:24.619 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:57:24 smithi089 bash[72894]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:57:24] "GET /metrics HTTP/1.1" 200 33866 "" "Prometheus/2.43.0" 2024-07-15T06:57:24.803 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:57:24 smithi134 bash[60729]: ts=2024-07-15T06:57:24.454Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:57:24.803 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:57:24 smithi134 bash[60729]: ts=2024-07-15T06:57:24.455Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:57:26.053 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:25 smithi134 bash[24068]: cluster 2024-07-15T06:57:24.704846+0000 mgr.y (mgr.25244) 27 : cluster [DBG] pgmap v13: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:57:26.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:25 smithi089 bash[79810]: cluster 2024-07-15T06:57:24.704846+0000 mgr.y (mgr.25244) 27 : cluster [DBG] pgmap v13: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:57:26.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:25 smithi089 bash[31372]: cluster 2024-07-15T06:57:24.704846+0000 mgr.y (mgr.25244) 27 : cluster [DBG] pgmap v13: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:57:28.053 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:27 smithi134 bash[24068]: cluster 2024-07-15T06:57:26.705903+0000 mgr.y (mgr.25244) 28 : cluster [DBG] pgmap v14: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:57:28.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:27 smithi089 bash[79810]: cluster 2024-07-15T06:57:26.705903+0000 mgr.y (mgr.25244) 28 : cluster [DBG] pgmap v14: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:57:28.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:27 smithi089 bash[31372]: cluster 2024-07-15T06:57:26.705903+0000 mgr.y (mgr.25244) 28 : cluster [DBG] pgmap v14: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:57:29.053 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:57:28 smithi134 bash[60729]: ts=2024-07-15T06:57:28.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1102-g9a202024 (9a2020246f85463d4fdb115c94aa40b463caa1d2) quincy (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1102-g9a202024 (9a2020246f85463d4fdb115c94aa40b463caa1d2) quincy (stable)\", cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:57:30.053 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:29 smithi134 bash[24068]: cluster 2024-07-15T06:57:28.706566+0000 mgr.y (mgr.25244) 29 : cluster [DBG] pgmap v15: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:57:30.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:29 smithi089 bash[79810]: cluster 2024-07-15T06:57:28.706566+0000 mgr.y (mgr.25244) 29 : cluster [DBG] pgmap v15: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:57:30.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:29 smithi089 bash[31372]: cluster 2024-07-15T06:57:28.706566+0000 mgr.y (mgr.25244) 29 : cluster [DBG] pgmap v15: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:57:32.053 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:31 smithi134 bash[24068]: cluster 2024-07-15T06:57:30.707222+0000 mgr.y (mgr.25244) 30 : cluster [DBG] pgmap v16: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:57:32.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:31 smithi089 bash[79810]: cluster 2024-07-15T06:57:30.707222+0000 mgr.y (mgr.25244) 30 : cluster [DBG] pgmap v16: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:57:32.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:31 smithi089 bash[31372]: cluster 2024-07-15T06:57:30.707222+0000 mgr.y (mgr.25244) 30 : cluster [DBG] pgmap v16: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:57:32.803 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:57:32 smithi134 bash[60729]: ts=2024-07-15T06:57:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:57:33.774 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:57:33 smithi134 bash[60729]: ts=2024-07-15T06:57:33.509Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi134\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi134\", job=\"node\", machine=\"x86_64\", nodename=\"smithi134\", release=\"5.15.0-113-generic\", sysname=\"Linux\", version=\"#123-Ubuntu SMP Mon Jun 10 08:16:17 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", domainname=\"(none)\", instance=\"smithi134\", job=\"node\", machine=\"x86_64\", nodename=\"smithi134\", release=\"5.15.0-113-generic\", sysname=\"Linux\", version=\"#123-Ubuntu SMP Mon Jun 10 08:16:17 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:57:34.053 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:33 smithi134 bash[24068]: cluster 2024-07-15T06:57:32.708299+0000 mgr.y (mgr.25244) 31 : cluster [DBG] pgmap v17: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:57:34.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:33 smithi089 bash[79810]: cluster 2024-07-15T06:57:32.708299+0000 mgr.y (mgr.25244) 31 : cluster [DBG] pgmap v17: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:57:34.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:33 smithi089 bash[31372]: cluster 2024-07-15T06:57:32.708299+0000 mgr.y (mgr.25244) 31 : cluster [DBG] pgmap v17: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:57:34.619 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:57:34 smithi089 bash[72894]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:57:34] "GET /metrics HTTP/1.1" 200 33866 "" "Prometheus/2.43.0" 2024-07-15T06:57:34.783 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:57:34 smithi134 bash[60729]: ts=2024-07-15T06:57:34.454Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:57:34.783 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:57:34 smithi134 bash[60729]: ts=2024-07-15T06:57:34.455Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:57:35.053 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:34 smithi134 bash[24068]: audit 2024-07-15T06:57:34.772597+0000 mon.c (mon.2) 99 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:57:35.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:34 smithi089 bash[79810]: audit 2024-07-15T06:57:34.772597+0000 mon.c (mon.2) 99 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:57:35.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:34 smithi089 bash[31372]: audit 2024-07-15T06:57:34.772597+0000 mon.c (mon.2) 99 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:57:36.053 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:35 smithi134 bash[24068]: cluster 2024-07-15T06:57:34.708970+0000 mgr.y (mgr.25244) 32 : cluster [DBG] pgmap v18: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:57:36.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:35 smithi089 bash[79810]: cluster 2024-07-15T06:57:34.708970+0000 mgr.y (mgr.25244) 32 : cluster [DBG] pgmap v18: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:57:36.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:35 smithi089 bash[31372]: cluster 2024-07-15T06:57:34.708970+0000 mgr.y (mgr.25244) 32 : cluster [DBG] pgmap v18: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:57:37.053 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:36 smithi134 bash[24068]: cluster 2024-07-15T06:57:36.710077+0000 mgr.y (mgr.25244) 33 : cluster [DBG] pgmap v19: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:57:37.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:36 smithi089 bash[79810]: cluster 2024-07-15T06:57:36.710077+0000 mgr.y (mgr.25244) 33 : cluster [DBG] pgmap v19: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:57:37.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:36 smithi089 bash[31372]: cluster 2024-07-15T06:57:36.710077+0000 mgr.y (mgr.25244) 33 : cluster [DBG] pgmap v19: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:57:39.053 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:57:38 smithi134 bash[60729]: ts=2024-07-15T06:57:38.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1102-g9a202024 (9a2020246f85463d4fdb115c94aa40b463caa1d2) quincy (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1102-g9a202024 (9a2020246f85463d4fdb115c94aa40b463caa1d2) quincy (stable)\", cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:57:40.053 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:39 smithi134 bash[24068]: cluster 2024-07-15T06:57:38.710666+0000 mgr.y (mgr.25244) 34 : cluster [DBG] pgmap v20: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:57:40.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:39 smithi089 bash[79810]: cluster 2024-07-15T06:57:38.710666+0000 mgr.y (mgr.25244) 34 : cluster [DBG] pgmap v20: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:57:40.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:39 smithi089 bash[31372]: cluster 2024-07-15T06:57:38.710666+0000 mgr.y (mgr.25244) 34 : cluster [DBG] pgmap v20: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:57:42.053 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:41 smithi134 bash[24068]: cluster 2024-07-15T06:57:40.711275+0000 mgr.y (mgr.25244) 35 : cluster [DBG] pgmap v21: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:57:42.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:41 smithi089 bash[79810]: cluster 2024-07-15T06:57:40.711275+0000 mgr.y (mgr.25244) 35 : cluster [DBG] pgmap v21: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:57:42.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:41 smithi089 bash[31372]: cluster 2024-07-15T06:57:40.711275+0000 mgr.y (mgr.25244) 35 : cluster [DBG] pgmap v21: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:57:42.803 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:57:42 smithi134 bash[60729]: ts=2024-07-15T06:57:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:57:43.772 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:57:43 smithi134 bash[60729]: ts=2024-07-15T06:57:43.508Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi134\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi134\", job=\"node\", machine=\"x86_64\", nodename=\"smithi134\", release=\"5.15.0-113-generic\", sysname=\"Linux\", version=\"#123-Ubuntu SMP Mon Jun 10 08:16:17 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", domainname=\"(none)\", instance=\"smithi134\", job=\"node\", machine=\"x86_64\", nodename=\"smithi134\", release=\"5.15.0-113-generic\", sysname=\"Linux\", version=\"#123-Ubuntu SMP Mon Jun 10 08:16:17 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:57:44.053 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:43 smithi134 bash[24068]: cluster 2024-07-15T06:57:42.712299+0000 mgr.y (mgr.25244) 36 : cluster [DBG] pgmap v22: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:57:44.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:43 smithi089 bash[79810]: cluster 2024-07-15T06:57:42.712299+0000 mgr.y (mgr.25244) 36 : cluster [DBG] pgmap v22: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:57:44.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:43 smithi089 bash[31372]: cluster 2024-07-15T06:57:42.712299+0000 mgr.y (mgr.25244) 36 : cluster [DBG] pgmap v22: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:57:44.619 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:57:44 smithi089 bash[72894]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:57:44] "GET /metrics HTTP/1.1" 200 33863 "" "Prometheus/2.43.0" 2024-07-15T06:57:44.803 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:57:44 smithi134 bash[60729]: ts=2024-07-15T06:57:44.454Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:57:44.804 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:57:44 smithi134 bash[60729]: ts=2024-07-15T06:57:44.455Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:57:46.053 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:45 smithi134 bash[24068]: cluster 2024-07-15T06:57:44.712954+0000 mgr.y (mgr.25244) 37 : cluster [DBG] pgmap v23: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:57:46.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:45 smithi089 bash[79810]: cluster 2024-07-15T06:57:44.712954+0000 mgr.y (mgr.25244) 37 : cluster [DBG] pgmap v23: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:57:46.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:45 smithi089 bash[31372]: cluster 2024-07-15T06:57:44.712954+0000 mgr.y (mgr.25244) 37 : cluster [DBG] pgmap v23: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:57:48.053 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:47 smithi134 bash[24068]: cluster 2024-07-15T06:57:46.714039+0000 mgr.y (mgr.25244) 38 : cluster [DBG] pgmap v24: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:57:48.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:47 smithi089 bash[79810]: cluster 2024-07-15T06:57:46.714039+0000 mgr.y (mgr.25244) 38 : cluster [DBG] pgmap v24: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:57:48.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:47 smithi089 bash[31372]: cluster 2024-07-15T06:57:46.714039+0000 mgr.y (mgr.25244) 38 : cluster [DBG] pgmap v24: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:57:49.053 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:57:48 smithi134 bash[60729]: ts=2024-07-15T06:57:48.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1102-g9a202024 (9a2020246f85463d4fdb115c94aa40b463caa1d2) quincy (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1102-g9a202024 (9a2020246f85463d4fdb115c94aa40b463caa1d2) quincy (stable)\", cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:57:50.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:49 smithi134 bash[24068]: cluster 2024-07-15T06:57:48.714690+0000 mgr.y (mgr.25244) 39 : cluster [DBG] pgmap v25: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:57:50.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:49 smithi134 bash[24068]: audit 2024-07-15T06:57:49.773199+0000 mon.c (mon.2) 100 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:57:50.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:49 smithi089 bash[79810]: cluster 2024-07-15T06:57:48.714690+0000 mgr.y (mgr.25244) 39 : cluster [DBG] pgmap v25: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:57:50.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:49 smithi089 bash[79810]: audit 2024-07-15T06:57:49.773199+0000 mon.c (mon.2) 100 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:57:50.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:49 smithi089 bash[31372]: cluster 2024-07-15T06:57:48.714690+0000 mgr.y (mgr.25244) 39 : cluster [DBG] pgmap v25: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:57:50.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:49 smithi089 bash[31372]: audit 2024-07-15T06:57:49.773199+0000 mon.c (mon.2) 100 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:57:52.053 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:51 smithi134 bash[24068]: cluster 2024-07-15T06:57:50.715456+0000 mgr.y (mgr.25244) 40 : cluster [DBG] pgmap v26: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:57:52.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:51 smithi089 bash[79810]: cluster 2024-07-15T06:57:50.715456+0000 mgr.y (mgr.25244) 40 : cluster [DBG] pgmap v26: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:57:52.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:51 smithi089 bash[31372]: cluster 2024-07-15T06:57:50.715456+0000 mgr.y (mgr.25244) 40 : cluster [DBG] pgmap v26: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:57:52.803 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:57:52 smithi134 bash[60729]: ts=2024-07-15T06:57:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:57:53.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:52 smithi089 bash[79810]: cluster 2024-07-15T06:57:52.716532+0000 mgr.y (mgr.25244) 41 : cluster [DBG] pgmap v27: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:57:53.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:52 smithi089 bash[31372]: cluster 2024-07-15T06:57:52.716532+0000 mgr.y (mgr.25244) 41 : cluster [DBG] pgmap v27: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:57:53.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:52 smithi134 bash[24068]: cluster 2024-07-15T06:57:52.716532+0000 mgr.y (mgr.25244) 41 : cluster [DBG] pgmap v27: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:57:53.804 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:57:53 smithi134 bash[60729]: ts=2024-07-15T06:57:53.508Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi134\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi134\", job=\"node\", machine=\"x86_64\", nodename=\"smithi134\", release=\"5.15.0-113-generic\", sysname=\"Linux\", version=\"#123-Ubuntu SMP Mon Jun 10 08:16:17 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", domainname=\"(none)\", instance=\"smithi134\", job=\"node\", machine=\"x86_64\", nodename=\"smithi134\", release=\"5.15.0-113-generic\", sysname=\"Linux\", version=\"#123-Ubuntu SMP Mon Jun 10 08:16:17 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:57:54.619 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:57:54 smithi089 bash[72894]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:57:54] "GET /metrics HTTP/1.1" 200 33865 "" "Prometheus/2.43.0" 2024-07-15T06:57:54.804 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:57:54 smithi134 bash[60729]: ts=2024-07-15T06:57:54.453Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:57:54.804 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:57:54 smithi134 bash[60729]: ts=2024-07-15T06:57:54.454Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:57:56.053 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:55 smithi134 bash[24068]: cluster 2024-07-15T06:57:54.717162+0000 mgr.y (mgr.25244) 42 : cluster [DBG] pgmap v28: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:57:56.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:55 smithi089 bash[79810]: cluster 2024-07-15T06:57:54.717162+0000 mgr.y (mgr.25244) 42 : cluster [DBG] pgmap v28: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:57:56.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:55 smithi089 bash[31372]: cluster 2024-07-15T06:57:54.717162+0000 mgr.y (mgr.25244) 42 : cluster [DBG] pgmap v28: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:57:58.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:57 smithi134 bash[24068]: cluster 2024-07-15T06:57:56.718230+0000 mgr.y (mgr.25244) 43 : cluster [DBG] pgmap v29: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:57:58.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:57 smithi089 bash[79810]: cluster 2024-07-15T06:57:56.718230+0000 mgr.y (mgr.25244) 43 : cluster [DBG] pgmap v29: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:57:58.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:57 smithi089 bash[31372]: cluster 2024-07-15T06:57:56.718230+0000 mgr.y (mgr.25244) 43 : cluster [DBG] pgmap v29: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:57:59.054 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:57:58 smithi134 bash[60729]: ts=2024-07-15T06:57:58.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1102-g9a202024 (9a2020246f85463d4fdb115c94aa40b463caa1d2) quincy (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1102-g9a202024 (9a2020246f85463d4fdb115c94aa40b463caa1d2) quincy (stable)\", cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:58:00.053 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:57:59 smithi134 bash[24068]: cluster 2024-07-15T06:57:58.718938+0000 mgr.y (mgr.25244) 44 : cluster [DBG] pgmap v30: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:58:00.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:57:59 smithi089 bash[31372]: cluster 2024-07-15T06:57:58.718938+0000 mgr.y (mgr.25244) 44 : cluster [DBG] pgmap v30: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:58:00.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:57:59 smithi089 bash[79810]: cluster 2024-07-15T06:57:58.718938+0000 mgr.y (mgr.25244) 44 : cluster [DBG] pgmap v30: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:58:02.053 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:58:01 smithi134 bash[24068]: cluster 2024-07-15T06:58:00.719647+0000 mgr.y (mgr.25244) 45 : cluster [DBG] pgmap v31: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:58:02.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:01 smithi089 bash[31372]: cluster 2024-07-15T06:58:00.719647+0000 mgr.y (mgr.25244) 45 : cluster [DBG] pgmap v31: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:58:02.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:01 smithi089 bash[79810]: cluster 2024-07-15T06:58:00.719647+0000 mgr.y (mgr.25244) 45 : cluster [DBG] pgmap v31: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:58:02.804 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:58:02 smithi134 bash[60729]: ts=2024-07-15T06:58:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:58:03.786 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:58:03 smithi134 bash[60729]: ts=2024-07-15T06:58:03.508Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi134\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi134\", job=\"node\", machine=\"x86_64\", nodename=\"smithi134\", release=\"5.15.0-113-generic\", sysname=\"Linux\", version=\"#123-Ubuntu SMP Mon Jun 10 08:16:17 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", domainname=\"(none)\", instance=\"smithi134\", job=\"node\", machine=\"x86_64\", nodename=\"smithi134\", release=\"5.15.0-113-generic\", sysname=\"Linux\", version=\"#123-Ubuntu SMP Mon Jun 10 08:16:17 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:58:04.053 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:58:03 smithi134 bash[24068]: cluster 2024-07-15T06:58:02.720757+0000 mgr.y (mgr.25244) 46 : cluster [DBG] pgmap v32: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:58:04.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:03 smithi089 bash[79810]: cluster 2024-07-15T06:58:02.720757+0000 mgr.y (mgr.25244) 46 : cluster [DBG] pgmap v32: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:58:04.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:03 smithi089 bash[31372]: cluster 2024-07-15T06:58:02.720757+0000 mgr.y (mgr.25244) 46 : cluster [DBG] pgmap v32: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:58:04.619 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:58:04 smithi089 bash[72894]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:58:04] "GET /metrics HTTP/1.1" 200 33865 "" "Prometheus/2.43.0" 2024-07-15T06:58:04.796 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:58:04 smithi134 bash[60729]: ts=2024-07-15T06:58:04.454Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:58:04.796 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:58:04 smithi134 bash[60729]: ts=2024-07-15T06:58:04.455Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:58:05.053 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:58:04 smithi134 bash[24068]: audit 2024-07-15T06:58:04.784421+0000 mon.c (mon.2) 101 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:58:05.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:04 smithi089 bash[79810]: audit 2024-07-15T06:58:04.784421+0000 mon.c (mon.2) 101 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:58:05.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:04 smithi089 bash[31372]: audit 2024-07-15T06:58:04.784421+0000 mon.c (mon.2) 101 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:58:06.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:58:05 smithi134 bash[24068]: cluster 2024-07-15T06:58:04.721380+0000 mgr.y (mgr.25244) 47 : cluster [DBG] pgmap v33: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:58:06.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:05 smithi089 bash[79810]: cluster 2024-07-15T06:58:04.721380+0000 mgr.y (mgr.25244) 47 : cluster [DBG] pgmap v33: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:58:06.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:05 smithi089 bash[31372]: cluster 2024-07-15T06:58:04.721380+0000 mgr.y (mgr.25244) 47 : cluster [DBG] pgmap v33: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:58:07.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:06 smithi089 bash[79810]: cluster 2024-07-15T06:58:06.722540+0000 mgr.y (mgr.25244) 48 : cluster [DBG] pgmap v34: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:58:07.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:06 smithi089 bash[31372]: cluster 2024-07-15T06:58:06.722540+0000 mgr.y (mgr.25244) 48 : cluster [DBG] pgmap v34: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:58:07.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:58:06 smithi134 bash[24068]: cluster 2024-07-15T06:58:06.722540+0000 mgr.y (mgr.25244) 48 : cluster [DBG] pgmap v34: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:58:09.054 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:58:08 smithi134 bash[60729]: ts=2024-07-15T06:58:08.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1102-g9a202024 (9a2020246f85463d4fdb115c94aa40b463caa1d2) quincy (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1102-g9a202024 (9a2020246f85463d4fdb115c94aa40b463caa1d2) quincy (stable)\", cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:58:10.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:58:09 smithi134 bash[24068]: cluster 2024-07-15T06:58:08.723161+0000 mgr.y (mgr.25244) 49 : cluster [DBG] pgmap v35: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:58:10.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:09 smithi089 bash[79810]: cluster 2024-07-15T06:58:08.723161+0000 mgr.y (mgr.25244) 49 : cluster [DBG] pgmap v35: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:58:10.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:09 smithi089 bash[31372]: cluster 2024-07-15T06:58:08.723161+0000 mgr.y (mgr.25244) 49 : cluster [DBG] pgmap v35: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:58:12.053 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:58:11 smithi134 bash[24068]: cluster 2024-07-15T06:58:10.723773+0000 mgr.y (mgr.25244) 50 : cluster [DBG] pgmap v36: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:58:12.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:11 smithi089 bash[79810]: cluster 2024-07-15T06:58:10.723773+0000 mgr.y (mgr.25244) 50 : cluster [DBG] pgmap v36: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:58:12.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:11 smithi089 bash[31372]: cluster 2024-07-15T06:58:10.723773+0000 mgr.y (mgr.25244) 50 : cluster [DBG] pgmap v36: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:58:12.804 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:58:12 smithi134 bash[60729]: ts=2024-07-15T06:58:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:58:13.783 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:58:13 smithi134 bash[60729]: ts=2024-07-15T06:58:13.508Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi134\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi134\", job=\"node\", machine=\"x86_64\", nodename=\"smithi134\", release=\"5.15.0-113-generic\", sysname=\"Linux\", version=\"#123-Ubuntu SMP Mon Jun 10 08:16:17 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", domainname=\"(none)\", instance=\"smithi134\", job=\"node\", machine=\"x86_64\", nodename=\"smithi134\", release=\"5.15.0-113-generic\", sysname=\"Linux\", version=\"#123-Ubuntu SMP Mon Jun 10 08:16:17 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:58:14.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:58:13 smithi134 bash[24068]: cluster 2024-07-15T06:58:12.724873+0000 mgr.y (mgr.25244) 51 : cluster [DBG] pgmap v37: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:58:14.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:13 smithi089 bash[79810]: cluster 2024-07-15T06:58:12.724873+0000 mgr.y (mgr.25244) 51 : cluster [DBG] pgmap v37: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:58:14.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:13 smithi089 bash[31372]: cluster 2024-07-15T06:58:12.724873+0000 mgr.y (mgr.25244) 51 : cluster [DBG] pgmap v37: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:58:14.619 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:58:14 smithi089 bash[72894]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:58:14] "GET /metrics HTTP/1.1" 200 33856 "" "Prometheus/2.43.0" 2024-07-15T06:58:14.804 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:58:14 smithi134 bash[60729]: ts=2024-07-15T06:58:14.453Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:58:14.804 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:58:14 smithi134 bash[60729]: ts=2024-07-15T06:58:14.454Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:58:16.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:58:15 smithi134 bash[24068]: cluster 2024-07-15T06:58:14.725618+0000 mgr.y (mgr.25244) 52 : cluster [DBG] pgmap v38: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:58:16.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:15 smithi089 bash[79810]: cluster 2024-07-15T06:58:14.725618+0000 mgr.y (mgr.25244) 52 : cluster [DBG] pgmap v38: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:58:16.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:15 smithi089 bash[31372]: cluster 2024-07-15T06:58:14.725618+0000 mgr.y (mgr.25244) 52 : cluster [DBG] pgmap v38: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:58:18.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:58:17 smithi134 bash[24068]: cluster 2024-07-15T06:58:16.726814+0000 mgr.y (mgr.25244) 53 : cluster [DBG] pgmap v39: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:58:18.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:17 smithi089 bash[79810]: cluster 2024-07-15T06:58:16.726814+0000 mgr.y (mgr.25244) 53 : cluster [DBG] pgmap v39: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:58:18.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:17 smithi089 bash[31372]: cluster 2024-07-15T06:58:16.726814+0000 mgr.y (mgr.25244) 53 : cluster [DBG] pgmap v39: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:58:19.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:58:18 smithi134 bash[24068]: cluster 2024-07-15T06:58:18.727606+0000 mgr.y (mgr.25244) 54 : cluster [DBG] pgmap v40: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:58:19.054 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:58:18 smithi134 bash[60729]: ts=2024-07-15T06:58:18.662Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1102-g9a202024 (9a2020246f85463d4fdb115c94aa40b463caa1d2) quincy (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1102-g9a202024 (9a2020246f85463d4fdb115c94aa40b463caa1d2) quincy (stable)\", cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:58:19.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:18 smithi089 bash[79810]: cluster 2024-07-15T06:58:18.727606+0000 mgr.y (mgr.25244) 54 : cluster [DBG] pgmap v40: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:58:19.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:18 smithi089 bash[31372]: cluster 2024-07-15T06:58:18.727606+0000 mgr.y (mgr.25244) 54 : cluster [DBG] pgmap v40: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:58:20.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:58:19 smithi134 bash[24068]: audit 2024-07-15T06:58:19.784875+0000 mon.c (mon.2) 102 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:58:20.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:19 smithi089 bash[79810]: audit 2024-07-15T06:58:19.784875+0000 mon.c (mon.2) 102 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:58:20.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:19 smithi089 bash[31372]: audit 2024-07-15T06:58:19.784875+0000 mon.c (mon.2) 102 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:58:21.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:20 smithi089 bash[31372]: cluster 2024-07-15T06:58:20.728328+0000 mgr.y (mgr.25244) 55 : cluster [DBG] pgmap v41: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:58:21.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:20 smithi089 bash[79810]: cluster 2024-07-15T06:58:20.728328+0000 mgr.y (mgr.25244) 55 : cluster [DBG] pgmap v41: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:58:21.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:58:20 smithi134 bash[24068]: cluster 2024-07-15T06:58:20.728328+0000 mgr.y (mgr.25244) 55 : cluster [DBG] pgmap v41: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:58:22.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:58:22 smithi134 bash[24068]: audit 2024-07-15T06:58:22.382029+0000 mon.c (mon.2) 103 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:58:22.804 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:58:22 smithi134 bash[60729]: ts=2024-07-15T06:58:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:58:22.869 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:22 smithi089 bash[79810]: audit 2024-07-15T06:58:22.382029+0000 mon.c (mon.2) 103 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:58:22.870 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:22 smithi089 bash[31372]: audit 2024-07-15T06:58:22.382029+0000 mon.c (mon.2) 103 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:58:23.804 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:58:23 smithi134 bash[60729]: ts=2024-07-15T06:58:23.507Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi134\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi134\", job=\"node\", machine=\"x86_64\", nodename=\"smithi134\", release=\"5.15.0-113-generic\", sysname=\"Linux\", version=\"#123-Ubuntu SMP Mon Jun 10 08:16:17 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", domainname=\"(none)\", instance=\"smithi134\", job=\"node\", machine=\"x86_64\", nodename=\"smithi134\", release=\"5.15.0-113-generic\", sysname=\"Linux\", version=\"#123-Ubuntu SMP Mon Jun 10 08:16:17 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:58:23.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:58:23 smithi134 bash[24068]: cluster 2024-07-15T06:58:22.729101+0000 mgr.y (mgr.25244) 56 : cluster [DBG] pgmap v42: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:58:23.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:58:23 smithi134 bash[24068]: audit 2024-07-15T06:58:22.752851+0000 mon.a (mon.0) 41 : audit [INF] from='mgr.25244 ' entity='mgr.y' 2024-07-15T06:58:23.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:58:23 smithi134 bash[24068]: audit 2024-07-15T06:58:22.759984+0000 mon.a (mon.0) 42 : audit [INF] from='mgr.25244 ' entity='mgr.y' 2024-07-15T06:58:24.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:23 smithi089 bash[79810]: cluster 2024-07-15T06:58:22.729101+0000 mgr.y (mgr.25244) 56 : cluster [DBG] pgmap v42: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:58:24.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:23 smithi089 bash[79810]: audit 2024-07-15T06:58:22.752851+0000 mon.a (mon.0) 41 : audit [INF] from='mgr.25244 ' entity='mgr.y' 2024-07-15T06:58:24.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:23 smithi089 bash[79810]: audit 2024-07-15T06:58:22.759984+0000 mon.a (mon.0) 42 : audit [INF] from='mgr.25244 ' entity='mgr.y' 2024-07-15T06:58:24.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:23 smithi089 bash[31372]: cluster 2024-07-15T06:58:22.729101+0000 mgr.y (mgr.25244) 56 : cluster [DBG] pgmap v42: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:58:24.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:23 smithi089 bash[31372]: audit 2024-07-15T06:58:22.752851+0000 mon.a (mon.0) 41 : audit [INF] from='mgr.25244 ' entity='mgr.y' 2024-07-15T06:58:24.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:23 smithi089 bash[31372]: audit 2024-07-15T06:58:22.759984+0000 mon.a (mon.0) 42 : audit [INF] from='mgr.25244 ' entity='mgr.y' 2024-07-15T06:58:24.619 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:58:24 smithi089 bash[72894]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:58:24] "GET /metrics HTTP/1.1" 200 33862 "" "Prometheus/2.43.0" 2024-07-15T06:58:24.804 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:58:24 smithi134 bash[60729]: ts=2024-07-15T06:58:24.453Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:58:24.804 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:58:24 smithi134 bash[60729]: ts=2024-07-15T06:58:24.454Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:58:26.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:58:25 smithi134 bash[24068]: cluster 2024-07-15T06:58:24.729721+0000 mgr.y (mgr.25244) 57 : cluster [DBG] pgmap v43: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:58:26.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:25 smithi089 bash[79810]: cluster 2024-07-15T06:58:24.729721+0000 mgr.y (mgr.25244) 57 : cluster [DBG] pgmap v43: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:58:26.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:25 smithi089 bash[31372]: cluster 2024-07-15T06:58:24.729721+0000 mgr.y (mgr.25244) 57 : cluster [DBG] pgmap v43: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:58:27.638 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -e sha1=6ea7aaa9f93d34dc4d21862afe00140dfddbd706 -- bash -c 'ceph orch upgrade resume' 2024-07-15T06:58:28.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:58:27 smithi134 bash[24068]: cluster 2024-07-15T06:58:26.730780+0000 mgr.y (mgr.25244) 58 : cluster [DBG] pgmap v44: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:58:28.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:27 smithi089 bash[31372]: cluster 2024-07-15T06:58:26.730780+0000 mgr.y (mgr.25244) 58 : cluster [DBG] pgmap v44: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:58:28.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:27 smithi089 bash[79810]: cluster 2024-07-15T06:58:26.730780+0000 mgr.y (mgr.25244) 58 : cluster [DBG] pgmap v44: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:58:29.054 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:58:28 smithi134 bash[60729]: ts=2024-07-15T06:58:28.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1102-g9a202024 (9a2020246f85463d4fdb115c94aa40b463caa1d2) quincy (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1102-g9a202024 (9a2020246f85463d4fdb115c94aa40b463caa1d2) quincy (stable)\", cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:58:29.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:58:29 smithi134 bash[24068]: audit 2024-07-15T06:58:28.245728+0000 mon.a (mon.0) 43 : audit [INF] from='mgr.25244 ' entity='mgr.y' 2024-07-15T06:58:29.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:58:29 smithi134 bash[24068]: audit 2024-07-15T06:58:28.255982+0000 mon.a (mon.0) 44 : audit [INF] from='mgr.25244 ' entity='mgr.y' 2024-07-15T06:58:29.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:58:29 smithi134 bash[24068]: cluster 2024-07-15T06:58:28.731159+0000 mgr.y (mgr.25244) 59 : cluster [DBG] pgmap v45: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:58:29.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:58:29 smithi134 bash[24068]: audit 2024-07-15T06:58:28.757455+0000 mon.a (mon.0) 45 : audit [INF] from='mgr.25244 ' entity='mgr.y' 2024-07-15T06:58:29.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:58:29 smithi134 bash[24068]: audit 2024-07-15T06:58:28.763007+0000 mon.a (mon.0) 46 : audit [INF] from='mgr.25244 ' entity='mgr.y' 2024-07-15T06:58:29.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:58:29 smithi134 bash[24068]: audit 2024-07-15T06:58:29.177262+0000 mon.c (mon.2) 104 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:58:29.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:58:29 smithi134 bash[24068]: audit 2024-07-15T06:58:29.178712+0000 mon.c (mon.2) 105 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:58:29.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:58:29 smithi134 bash[24068]: audit 2024-07-15T06:58:29.189489+0000 mon.a (mon.0) 47 : audit [INF] from='mgr.25244 ' entity='mgr.y' 2024-07-15T06:58:29.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:29 smithi089 bash[79810]: audit 2024-07-15T06:58:28.245728+0000 mon.a (mon.0) 43 : audit [INF] from='mgr.25244 ' entity='mgr.y' 2024-07-15T06:58:29.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:29 smithi089 bash[79810]: audit 2024-07-15T06:58:28.255982+0000 mon.a (mon.0) 44 : audit [INF] from='mgr.25244 ' entity='mgr.y' 2024-07-15T06:58:29.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:29 smithi089 bash[79810]: cluster 2024-07-15T06:58:28.731159+0000 mgr.y (mgr.25244) 59 : cluster [DBG] pgmap v45: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:58:29.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:29 smithi089 bash[79810]: audit 2024-07-15T06:58:28.757455+0000 mon.a (mon.0) 45 : audit [INF] from='mgr.25244 ' entity='mgr.y' 2024-07-15T06:58:29.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:29 smithi089 bash[79810]: audit 2024-07-15T06:58:28.763007+0000 mon.a (mon.0) 46 : audit [INF] from='mgr.25244 ' entity='mgr.y' 2024-07-15T06:58:29.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:29 smithi089 bash[79810]: audit 2024-07-15T06:58:29.177262+0000 mon.c (mon.2) 104 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:58:29.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:29 smithi089 bash[79810]: audit 2024-07-15T06:58:29.178712+0000 mon.c (mon.2) 105 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:58:29.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:29 smithi089 bash[79810]: audit 2024-07-15T06:58:29.189489+0000 mon.a (mon.0) 47 : audit [INF] from='mgr.25244 ' entity='mgr.y' 2024-07-15T06:58:29.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:29 smithi089 bash[31372]: audit 2024-07-15T06:58:28.245728+0000 mon.a (mon.0) 43 : audit [INF] from='mgr.25244 ' entity='mgr.y' 2024-07-15T06:58:29.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:29 smithi089 bash[31372]: audit 2024-07-15T06:58:28.255982+0000 mon.a (mon.0) 44 : audit [INF] from='mgr.25244 ' entity='mgr.y' 2024-07-15T06:58:29.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:29 smithi089 bash[31372]: cluster 2024-07-15T06:58:28.731159+0000 mgr.y (mgr.25244) 59 : cluster [DBG] pgmap v45: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:58:29.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:29 smithi089 bash[31372]: audit 2024-07-15T06:58:28.757455+0000 mon.a (mon.0) 45 : audit [INF] from='mgr.25244 ' entity='mgr.y' 2024-07-15T06:58:29.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:29 smithi089 bash[31372]: audit 2024-07-15T06:58:28.763007+0000 mon.a (mon.0) 46 : audit [INF] from='mgr.25244 ' entity='mgr.y' 2024-07-15T06:58:29.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:29 smithi089 bash[31372]: audit 2024-07-15T06:58:29.177262+0000 mon.c (mon.2) 104 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:58:29.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:29 smithi089 bash[31372]: audit 2024-07-15T06:58:29.178712+0000 mon.c (mon.2) 105 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:58:29.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:29 smithi089 bash[31372]: audit 2024-07-15T06:58:29.189489+0000 mon.a (mon.0) 47 : audit [INF] from='mgr.25244 ' entity='mgr.y' 2024-07-15T06:58:32.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:58:31 smithi134 bash[24068]: cluster 2024-07-15T06:58:30.731740+0000 mgr.y (mgr.25244) 60 : cluster [DBG] pgmap v46: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:58:32.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:31 smithi089 bash[79810]: cluster 2024-07-15T06:58:30.731740+0000 mgr.y (mgr.25244) 60 : cluster [DBG] pgmap v46: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:58:32.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:31 smithi089 bash[31372]: cluster 2024-07-15T06:58:30.731740+0000 mgr.y (mgr.25244) 60 : cluster [DBG] pgmap v46: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:58:32.182 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T06:58:32.804 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:58:32 smithi134 bash[60729]: ts=2024-07-15T06:58:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:58:33.789 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:58:33 smithi134 bash[60729]: ts=2024-07-15T06:58:33.507Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi134\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi134\", job=\"node\", machine=\"x86_64\", nodename=\"smithi134\", release=\"5.15.0-113-generic\", sysname=\"Linux\", version=\"#123-Ubuntu SMP Mon Jun 10 08:16:17 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", domainname=\"(none)\", instance=\"smithi134\", job=\"node\", machine=\"x86_64\", nodename=\"smithi134\", release=\"5.15.0-113-generic\", sysname=\"Linux\", version=\"#123-Ubuntu SMP Mon Jun 10 08:16:17 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:58:34.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:58:33 smithi134 bash[24068]: cluster 2024-07-15T06:58:32.732808+0000 mgr.y (mgr.25244) 61 : cluster [DBG] pgmap v47: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:58:34.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:33 smithi089 bash[31372]: cluster 2024-07-15T06:58:32.732808+0000 mgr.y (mgr.25244) 61 : cluster [DBG] pgmap v47: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:58:34.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:33 smithi089 bash[79810]: cluster 2024-07-15T06:58:32.732808+0000 mgr.y (mgr.25244) 61 : cluster [DBG] pgmap v47: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:58:34.489 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:58:34 smithi089 bash[72894]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:58:34] "GET /metrics HTTP/1.1" 200 33862 "" "Prometheus/2.43.0" 2024-07-15T06:58:34.801 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:58:34 smithi134 bash[60729]: ts=2024-07-15T06:58:34.454Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:58:34.801 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:58:34 smithi134 bash[60729]: ts=2024-07-15T06:58:34.454Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:58:35.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:58:34 smithi134 bash[24068]: audit 2024-07-15T06:58:34.785240+0000 mon.c (mon.2) 106 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:58:35.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:34 smithi089 bash[79810]: audit 2024-07-15T06:58:34.785240+0000 mon.c (mon.2) 106 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:58:35.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:34 smithi089 bash[31372]: audit 2024-07-15T06:58:34.785240+0000 mon.c (mon.2) 106 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:58:35.971 INFO:teuthology.orchestra.run.smithi089.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:badc66b92d2a2cfbbce7f293b5accdc50f486d2b2d966876e46fe28d05f464f1 2024-07-15T06:58:35.984 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:35 smithi089 bash[79810]: cluster 2024-07-15T06:58:34.733428+0000 mgr.y (mgr.25244) 62 : cluster [DBG] pgmap v48: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:58:35.985 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:35 smithi089 bash[31372]: cluster 2024-07-15T06:58:34.733428+0000 mgr.y (mgr.25244) 62 : cluster [DBG] pgmap v48: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:58:36.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:58:35 smithi134 bash[24068]: cluster 2024-07-15T06:58:34.733428+0000 mgr.y (mgr.25244) 62 : cluster [DBG] pgmap v48: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:58:37.017 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:36 smithi089 bash[79810]: cluster 2024-07-15T06:58:35.818550+0000 mon.a (mon.0) 48 : cluster [DBG] mgrmap e40: y(active, since 91s), standbys: x 2024-07-15T06:58:37.018 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:36 smithi089 bash[79810]: audit 2024-07-15T06:58:35.966292+0000 mgr.y (mgr.25244) 63 : audit [DBG] from='client.25298 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T06:58:37.018 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:36 smithi089 bash[79810]: cephadm 2024-07-15T06:58:35.966831+0000 mgr.y (mgr.25244) 64 : cephadm [INF] Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:badc66b92d2a2cfbbce7f293b5accdc50f486d2b2d966876e46fe28d05f464f1 2024-07-15T06:58:37.018 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:36 smithi089 bash[79810]: audit 2024-07-15T06:58:35.975606+0000 mon.a (mon.0) 49 : audit [INF] from='mgr.25244 ' entity='mgr.y' 2024-07-15T06:58:37.018 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:36 smithi089 bash[79810]: audit 2024-07-15T06:58:35.977695+0000 mon.c (mon.2) 107 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:58:37.018 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:36 smithi089 bash[79810]: audit 2024-07-15T06:58:35.980690+0000 mon.c (mon.2) 108 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:58:37.018 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:36 smithi089 bash[79810]: audit 2024-07-15T06:58:35.981789+0000 mon.c (mon.2) 109 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:58:37.018 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:36 smithi089 bash[79810]: audit 2024-07-15T06:58:35.992019+0000 mon.a (mon.0) 50 : audit [INF] from='mgr.25244 ' entity='mgr.y' 2024-07-15T06:58:37.018 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:36 smithi089 bash[79810]: audit 2024-07-15T06:58:36.072139+0000 mon.c (mon.2) 110 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:58:37.018 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:36 smithi089 bash[79810]: audit 2024-07-15T06:58:36.075335+0000 mon.c (mon.2) 111 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-15T06:58:37.018 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:36 smithi089 bash[79810]: audit 2024-07-15T06:58:36.076397+0000 mon.c (mon.2) 112 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-07-15T06:58:37.018 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:36 smithi089 bash[79810]: audit 2024-07-15T06:58:36.077027+0000 mon.c (mon.2) 113 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-07-15T06:58:37.019 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:36 smithi089 bash[79810]: cephadm 2024-07-15T06:58:36.077487+0000 mgr.y (mgr.25244) 65 : cephadm [INF] Upgrade: It appears safe to stop mon.c 2024-07-15T06:58:37.019 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:36 smithi089 bash[79810]: cluster 2024-07-15T06:58:36.734542+0000 mgr.y (mgr.25244) 66 : cluster [DBG] pgmap v49: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-07-15T06:58:37.019 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:36 smithi089 bash[31372]: cluster 2024-07-15T06:58:35.818550+0000 mon.a (mon.0) 48 : cluster [DBG] mgrmap e40: y(active, since 91s), standbys: x 2024-07-15T06:58:37.019 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:36 smithi089 bash[31372]: audit 2024-07-15T06:58:35.966292+0000 mgr.y (mgr.25244) 63 : audit [DBG] from='client.25298 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T06:58:37.019 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:36 smithi089 bash[31372]: cephadm 2024-07-15T06:58:35.966831+0000 mgr.y (mgr.25244) 64 : cephadm [INF] Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:badc66b92d2a2cfbbce7f293b5accdc50f486d2b2d966876e46fe28d05f464f1 2024-07-15T06:58:37.019 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:36 smithi089 bash[31372]: audit 2024-07-15T06:58:35.975606+0000 mon.a (mon.0) 49 : audit [INF] from='mgr.25244 ' entity='mgr.y' 2024-07-15T06:58:37.019 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:36 smithi089 bash[31372]: audit 2024-07-15T06:58:35.977695+0000 mon.c (mon.2) 107 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:58:37.019 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:36 smithi089 bash[31372]: audit 2024-07-15T06:58:35.980690+0000 mon.c (mon.2) 108 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:58:37.019 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:36 smithi089 bash[31372]: audit 2024-07-15T06:58:35.981789+0000 mon.c (mon.2) 109 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:58:37.019 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:36 smithi089 bash[31372]: audit 2024-07-15T06:58:35.992019+0000 mon.a (mon.0) 50 : audit [INF] from='mgr.25244 ' entity='mgr.y' 2024-07-15T06:58:37.019 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:36 smithi089 bash[31372]: audit 2024-07-15T06:58:36.072139+0000 mon.c (mon.2) 110 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:58:37.020 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:36 smithi089 bash[31372]: audit 2024-07-15T06:58:36.075335+0000 mon.c (mon.2) 111 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-15T06:58:37.020 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:36 smithi089 bash[31372]: audit 2024-07-15T06:58:36.076397+0000 mon.c (mon.2) 112 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-07-15T06:58:37.020 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:36 smithi089 bash[31372]: audit 2024-07-15T06:58:36.077027+0000 mon.c (mon.2) 113 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-07-15T06:58:37.020 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:36 smithi089 bash[31372]: cephadm 2024-07-15T06:58:36.077487+0000 mgr.y (mgr.25244) 65 : cephadm [INF] Upgrade: It appears safe to stop mon.c 2024-07-15T06:58:37.020 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:36 smithi089 bash[31372]: cluster 2024-07-15T06:58:36.734542+0000 mgr.y (mgr.25244) 66 : cluster [DBG] pgmap v49: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-07-15T06:58:37.021 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -e sha1=6ea7aaa9f93d34dc4d21862afe00140dfddbd706 -- bash -c 'echo wait for majority of mons to upgrade' 2024-07-15T06:58:37.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:58:36 smithi134 bash[24068]: cluster 2024-07-15T06:58:35.818550+0000 mon.a (mon.0) 48 : cluster [DBG] mgrmap e40: y(active, since 91s), standbys: x 2024-07-15T06:58:37.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:58:36 smithi134 bash[24068]: audit 2024-07-15T06:58:35.966292+0000 mgr.y (mgr.25244) 63 : audit [DBG] from='client.25298 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T06:58:37.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:58:36 smithi134 bash[24068]: cephadm 2024-07-15T06:58:35.966831+0000 mgr.y (mgr.25244) 64 : cephadm [INF] Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:badc66b92d2a2cfbbce7f293b5accdc50f486d2b2d966876e46fe28d05f464f1 2024-07-15T06:58:37.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:58:36 smithi134 bash[24068]: audit 2024-07-15T06:58:35.975606+0000 mon.a (mon.0) 49 : audit [INF] from='mgr.25244 ' entity='mgr.y' 2024-07-15T06:58:37.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:58:36 smithi134 bash[24068]: audit 2024-07-15T06:58:35.977695+0000 mon.c (mon.2) 107 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:58:37.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:58:36 smithi134 bash[24068]: audit 2024-07-15T06:58:35.980690+0000 mon.c (mon.2) 108 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:58:37.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:58:36 smithi134 bash[24068]: audit 2024-07-15T06:58:35.981789+0000 mon.c (mon.2) 109 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:58:37.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:58:36 smithi134 bash[24068]: audit 2024-07-15T06:58:35.992019+0000 mon.a (mon.0) 50 : audit [INF] from='mgr.25244 ' entity='mgr.y' 2024-07-15T06:58:37.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:58:36 smithi134 bash[24068]: audit 2024-07-15T06:58:36.072139+0000 mon.c (mon.2) 110 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:58:37.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:58:36 smithi134 bash[24068]: audit 2024-07-15T06:58:36.075335+0000 mon.c (mon.2) 111 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-15T06:58:37.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:58:36 smithi134 bash[24068]: audit 2024-07-15T06:58:36.076397+0000 mon.c (mon.2) 112 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-07-15T06:58:37.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:58:36 smithi134 bash[24068]: audit 2024-07-15T06:58:36.077027+0000 mon.c (mon.2) 113 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-07-15T06:58:37.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:58:36 smithi134 bash[24068]: cephadm 2024-07-15T06:58:36.077487+0000 mgr.y (mgr.25244) 65 : cephadm [INF] Upgrade: It appears safe to stop mon.c 2024-07-15T06:58:37.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:58:36 smithi134 bash[24068]: cluster 2024-07-15T06:58:36.734542+0000 mgr.y (mgr.25244) 66 : cluster [DBG] pgmap v49: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-07-15T06:58:39.054 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:58:38 smithi134 bash[60729]: ts=2024-07-15T06:58:38.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1102-g9a202024 (9a2020246f85463d4fdb115c94aa40b463caa1d2) quincy (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1102-g9a202024 (9a2020246f85463d4fdb115c94aa40b463caa1d2) quincy (stable)\", cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:58:40.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:58:39 smithi134 bash[24068]: cluster 2024-07-15T06:58:38.735206+0000 mgr.y (mgr.25244) 67 : cluster [DBG] pgmap v50: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-07-15T06:58:40.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:58:39 smithi134 bash[24068]: cephadm 2024-07-15T06:58:38.794820+0000 mgr.y (mgr.25244) 68 : cephadm [INF] Upgrade: Updating mon.c 2024-07-15T06:58:40.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:58:39 smithi134 bash[24068]: audit 2024-07-15T06:58:38.804100+0000 mon.a (mon.0) 51 : audit [INF] from='mgr.25244 ' entity='mgr.y' 2024-07-15T06:58:40.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:58:39 smithi134 bash[24068]: audit 2024-07-15T06:58:38.810774+0000 mon.c (mon.2) 114 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-07-15T06:58:40.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:58:39 smithi134 bash[24068]: audit 2024-07-15T06:58:38.811866+0000 mon.c (mon.2) 115 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-07-15T06:58:40.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:58:39 smithi134 bash[24068]: audit 2024-07-15T06:58:38.812969+0000 mon.c (mon.2) 116 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:58:40.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:58:39 smithi134 bash[24068]: cephadm 2024-07-15T06:58:38.814531+0000 mgr.y (mgr.25244) 69 : cephadm [INF] Deploying daemon mon.c on smithi089 2024-07-15T06:58:40.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:39 smithi089 bash[79810]: cluster 2024-07-15T06:58:38.735206+0000 mgr.y (mgr.25244) 67 : cluster [DBG] pgmap v50: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-07-15T06:58:40.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:39 smithi089 bash[79810]: cephadm 2024-07-15T06:58:38.794820+0000 mgr.y (mgr.25244) 68 : cephadm [INF] Upgrade: Updating mon.c 2024-07-15T06:58:40.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:39 smithi089 bash[79810]: audit 2024-07-15T06:58:38.804100+0000 mon.a (mon.0) 51 : audit [INF] from='mgr.25244 ' entity='mgr.y' 2024-07-15T06:58:40.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:39 smithi089 bash[79810]: audit 2024-07-15T06:58:38.810774+0000 mon.c (mon.2) 114 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-07-15T06:58:40.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:39 smithi089 bash[79810]: audit 2024-07-15T06:58:38.811866+0000 mon.c (mon.2) 115 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-07-15T06:58:40.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:39 smithi089 bash[79810]: audit 2024-07-15T06:58:38.812969+0000 mon.c (mon.2) 116 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:58:40.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:39 smithi089 bash[79810]: cephadm 2024-07-15T06:58:38.814531+0000 mgr.y (mgr.25244) 69 : cephadm [INF] Deploying daemon mon.c on smithi089 2024-07-15T06:58:40.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:39 smithi089 bash[31372]: cluster 2024-07-15T06:58:38.735206+0000 mgr.y (mgr.25244) 67 : cluster [DBG] pgmap v50: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-07-15T06:58:40.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:39 smithi089 bash[31372]: cephadm 2024-07-15T06:58:38.794820+0000 mgr.y (mgr.25244) 68 : cephadm [INF] Upgrade: Updating mon.c 2024-07-15T06:58:40.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:39 smithi089 bash[31372]: audit 2024-07-15T06:58:38.804100+0000 mon.a (mon.0) 51 : audit [INF] from='mgr.25244 ' entity='mgr.y' 2024-07-15T06:58:40.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:39 smithi089 bash[31372]: audit 2024-07-15T06:58:38.810774+0000 mon.c (mon.2) 114 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-07-15T06:58:40.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:39 smithi089 bash[31372]: audit 2024-07-15T06:58:38.811866+0000 mon.c (mon.2) 115 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-07-15T06:58:40.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:39 smithi089 bash[31372]: audit 2024-07-15T06:58:38.812969+0000 mon.c (mon.2) 116 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:58:40.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:39 smithi089 bash[31372]: cephadm 2024-07-15T06:58:38.814531+0000 mgr.y (mgr.25244) 69 : cephadm [INF] Deploying daemon mon.c on smithi089 2024-07-15T06:58:41.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:40 smithi089 bash[79810]: cluster 2024-07-15T06:58:40.735868+0000 mgr.y (mgr.25244) 70 : cluster [DBG] pgmap v51: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-07-15T06:58:41.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:40 smithi089 bash[31372]: cluster 2024-07-15T06:58:40.735868+0000 mgr.y (mgr.25244) 70 : cluster [DBG] pgmap v51: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-07-15T06:58:41.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:58:40 smithi134 bash[24068]: cluster 2024-07-15T06:58:40.735868+0000 mgr.y (mgr.25244) 70 : cluster [DBG] pgmap v51: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-07-15T06:58:41.489 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:41 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:58:41.489 INFO:journalctl@ceph.osd.1.smithi089.stdout:Jul 15 06:58:41 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:58:41.489 INFO:journalctl@ceph.osd.3.smithi089.stdout:Jul 15 06:58:41 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:58:41.490 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:58:41 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:58:41.490 INFO:journalctl@ceph.osd.0.smithi089.stdout:Jul 15 06:58:41 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:58:41.493 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:58:41 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:58:41.493 INFO:journalctl@ceph.osd.2.smithi089.stdout:Jul 15 06:58:41 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:58:41.493 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 06:58:41 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:58:41.494 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:41 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:58:41.494 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:41 smithi089 systemd[1]: Stopping Ceph mon.c for 75ba02bc-4274-11ef-bcad-c7b262605968... 2024-07-15T06:58:41.778 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:41 smithi089 bash[31372]: debug 2024-07-15T06:58:41.600+0000 7fd4817ea640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2024-07-15T06:58:41.778 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:41 smithi089 bash[31372]: debug 2024-07-15T06:58:41.600+0000 7fd4817ea640 -1 mon.c@2(peon) e3 *** Got Signal Terminated *** 2024-07-15T06:58:41.778 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:58:41 smithi089 bash[72894]: [15/Jul/2024:06:58:41] ENGINE Bus STOPPING 2024-07-15T06:58:41.778 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:58:41 smithi089 bash[72894]: [15/Jul/2024:06:58:41] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-07-15T06:58:41.779 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:58:41 smithi089 bash[72894]: [15/Jul/2024:06:58:41] ENGINE Bus STOPPED 2024-07-15T06:58:41.779 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:58:41 smithi089 bash[72894]: [15/Jul/2024:06:58:41] ENGINE Bus STARTING 2024-07-15T06:58:42.057 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:58:41 smithi089 bash[72894]: [15/Jul/2024:06:58:41] ENGINE Serving on http://:::9283 2024-07-15T06:58:42.057 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:58:41 smithi089 bash[72894]: [15/Jul/2024:06:58:41] ENGINE Bus STARTED 2024-07-15T06:58:42.804 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:58:42 smithi134 bash[60729]: ts=2024-07-15T06:58:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:58:42.831 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:42 smithi089 bash[82077]: ceph-75ba02bc-4274-11ef-bcad-c7b262605968-mon-c 2024-07-15T06:58:42.832 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:42 smithi089 bash[82147]: Error response from daemon: No such container: ceph-75ba02bc-4274-11ef-bcad-c7b262605968-mon-c 2024-07-15T06:58:42.832 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:42 smithi089 systemd[1]: ceph-75ba02bc-4274-11ef-bcad-c7b262605968@mon.c.service: Deactivated successfully. 2024-07-15T06:58:42.832 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:42 smithi089 systemd[1]: Stopped Ceph mon.c for 75ba02bc-4274-11ef-bcad-c7b262605968. 2024-07-15T06:58:43.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:42 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:58:43.120 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:58:42 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:58:43.120 INFO:journalctl@ceph.osd.0.smithi089.stdout:Jul 15 06:58:42 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:58:43.120 INFO:journalctl@ceph.osd.1.smithi089.stdout:Jul 15 06:58:42 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:58:43.121 INFO:journalctl@ceph.osd.2.smithi089.stdout:Jul 15 06:58:42 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:58:43.121 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 06:58:42 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:58:43.121 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 06:58:42 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:58:43.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:42 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:58:43.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:42 smithi089 systemd[1]: Started Ceph mon.c for 75ba02bc-4274-11ef-bcad-c7b262605968. 2024-07-15T06:58:43.122 INFO:journalctl@ceph.osd.3.smithi089.stdout:Jul 15 06:58:42 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:58:43.172 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T06:58:43.804 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:58:43 smithi134 bash[60729]: ts=2024-07-15T06:58:43.508Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi134\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi134\", job=\"node\", machine=\"x86_64\", nodename=\"smithi134\", release=\"5.15.0-113-generic\", sysname=\"Linux\", version=\"#123-Ubuntu SMP Mon Jun 10 08:16:17 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", domainname=\"(none)\", instance=\"smithi134\", job=\"node\", machine=\"x86_64\", nodename=\"smithi134\", release=\"5.15.0-113-generic\", sysname=\"Linux\", version=\"#123-Ubuntu SMP Mon Jun 10 08:16:17 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:58:44.619 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:58:44 smithi089 bash[72894]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:58:44] "GET /metrics HTTP/1.1" 200 33859 "" "Prometheus/2.43.0" 2024-07-15T06:58:44.804 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:58:44 smithi134 bash[60729]: ts=2024-07-15T06:58:44.453Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:58:44.804 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:58:44 smithi134 bash[60729]: ts=2024-07-15T06:58:44.454Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:58:45.333 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.116+0000 7f4f3826bb00 0 set uid:gid to 167:167 (ceph:ceph) 2024-07-15T06:58:45.333 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.116+0000 7f4f3826bb00 0 ceph version 18.2.2-1609-g6ea7aaa9 (6ea7aaa9f93d34dc4d21862afe00140dfddbd706) reef (stable), process ceph-mon, pid 7 2024-07-15T06:58:45.333 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.116+0000 7f4f3826bb00 0 pidfile_write: ignore empty --pid-file 2024-07-15T06:58:45.333 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 0 load: jerasure load: lrc 2024-07-15T06:58:45.333 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: RocksDB version: 7.9.2 2024-07-15T06:58:45.333 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Git sha 0 2024-07-15T06:58:45.333 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Compile date 2024-07-05 15:13:19 2024-07-15T06:58:45.334 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: DB SUMMARY 2024-07-15T06:58:45.334 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: DB Session ID: EFDU2G3TA1P5N2J4L7T6 2024-07-15T06:58:45.334 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: CURRENT file: CURRENT 2024-07-15T06:58:45.334 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: IDENTITY file: IDENTITY 2024-07-15T06:58:45.334 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: MANIFEST file: MANIFEST-000009 size: 1659 Bytes 2024-07-15T06:58:45.334 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 1, files: 000036.sst 2024-07-15T06:58:45.334 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000034.log size: 4092277 ; 2024-07-15T06:58:45.334 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.error_if_exists: 0 2024-07-15T06:58:45.334 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.create_if_missing: 0 2024-07-15T06:58:45.334 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.paranoid_checks: 1 2024-07-15T06:58:45.334 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.flush_verify_memtable_count: 1 2024-07-15T06:58:45.335 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-07-15T06:58:45.335 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-07-15T06:58:45.335 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.env: 0x55c17e540ba0 2024-07-15T06:58:45.335 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.fs: PosixFileSystem 2024-07-15T06:58:45.335 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.info_log: 0x55c17f52ebe0 2024-07-15T06:58:45.335 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.max_file_opening_threads: 16 2024-07-15T06:58:45.335 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.statistics: (nil) 2024-07-15T06:58:45.335 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.use_fsync: 0 2024-07-15T06:58:45.335 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.max_log_file_size: 0 2024-07-15T06:58:45.335 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.max_manifest_file_size: 1073741824 2024-07-15T06:58:45.335 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.log_file_time_to_roll: 0 2024-07-15T06:58:45.336 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.keep_log_file_num: 1000 2024-07-15T06:58:45.336 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.recycle_log_file_num: 0 2024-07-15T06:58:45.336 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.allow_fallocate: 1 2024-07-15T06:58:45.336 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.allow_mmap_reads: 0 2024-07-15T06:58:45.336 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.allow_mmap_writes: 0 2024-07-15T06:58:45.336 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.use_direct_reads: 0 2024-07-15T06:58:45.336 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-07-15T06:58:45.336 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.create_missing_column_families: 0 2024-07-15T06:58:45.336 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.db_log_dir: 2024-07-15T06:58:45.336 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.wal_dir: 2024-07-15T06:58:45.336 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.table_cache_numshardbits: 6 2024-07-15T06:58:45.336 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.WAL_ttl_seconds: 0 2024-07-15T06:58:45.337 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.WAL_size_limit_MB: 0 2024-07-15T06:58:45.337 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-07-15T06:58:45.337 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.manifest_preallocation_size: 4194304 2024-07-15T06:58:45.337 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.is_fd_close_on_exec: 1 2024-07-15T06:58:45.337 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.advise_random_on_open: 1 2024-07-15T06:58:45.337 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.db_write_buffer_size: 0 2024-07-15T06:58:45.337 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.write_buffer_manager: 0x55c17f53cb40 2024-07-15T06:58:45.337 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.access_hint_on_compaction_start: 1 2024-07-15T06:58:45.337 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2024-07-15T06:58:45.337 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.use_adaptive_mutex: 0 2024-07-15T06:58:45.337 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.rate_limiter: (nil) 2024-07-15T06:58:45.337 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-07-15T06:58:45.337 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.wal_recovery_mode: 2 2024-07-15T06:58:45.338 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.enable_thread_tracking: 0 2024-07-15T06:58:45.338 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.enable_pipelined_write: 0 2024-07-15T06:58:45.338 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.unordered_write: 0 2024-07-15T06:58:45.338 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2024-07-15T06:58:45.338 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-07-15T06:58:45.338 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.write_thread_max_yield_usec: 100 2024-07-15T06:58:45.338 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2024-07-15T06:58:45.338 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.row_cache: None 2024-07-15T06:58:45.338 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.wal_filter: None 2024-07-15T06:58:45.338 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.avoid_flush_during_recovery: 0 2024-07-15T06:58:45.338 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.allow_ingest_behind: 0 2024-07-15T06:58:45.338 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.two_write_queues: 0 2024-07-15T06:58:45.339 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.manual_wal_flush: 0 2024-07-15T06:58:45.339 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.wal_compression: 0 2024-07-15T06:58:45.339 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.atomic_flush: 0 2024-07-15T06:58:45.339 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-07-15T06:58:45.339 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.persist_stats_to_disk: 0 2024-07-15T06:58:45.339 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.write_dbid_to_manifest: 0 2024-07-15T06:58:45.339 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.log_readahead_size: 0 2024-07-15T06:58:45.339 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.file_checksum_gen_factory: Unknown 2024-07-15T06:58:45.339 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.best_efforts_recovery: 0 2024-07-15T06:58:45.339 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-07-15T06:58:45.339 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-07-15T06:58:45.339 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.allow_data_in_errors: 0 2024-07-15T06:58:45.339 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.db_host_id: __hostname__ 2024-07-15T06:58:45.339 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.enforce_single_del_contracts: true 2024-07-15T06:58:45.340 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.max_background_jobs: 2 2024-07-15T06:58:45.340 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.max_background_compactions: -1 2024-07-15T06:58:45.341 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.max_subcompactions: 1 2024-07-15T06:58:45.341 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2024-07-15T06:58:45.341 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-07-15T06:58:45.341 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.delayed_write_rate : 16777216 2024-07-15T06:58:45.341 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.max_total_wal_size: 0 2024-07-15T06:58:45.341 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-07-15T06:58:45.341 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.stats_dump_period_sec: 600 2024-07-15T06:58:45.341 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.stats_persist_period_sec: 600 2024-07-15T06:58:45.342 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.stats_history_buffer_size: 1048576 2024-07-15T06:58:45.342 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.max_open_files: -1 2024-07-15T06:58:45.342 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.bytes_per_sync: 0 2024-07-15T06:58:45.342 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.wal_bytes_per_sync: 0 2024-07-15T06:58:45.342 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.strict_bytes_per_sync: 0 2024-07-15T06:58:45.342 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.compaction_readahead_size: 0 2024-07-15T06:58:45.342 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.max_background_flushes: -1 2024-07-15T06:58:45.342 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Compression algorithms supported: 2024-07-15T06:58:45.342 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: kZSTD supported: 0 2024-07-15T06:58:45.342 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: kXpressCompression supported: 0 2024-07-15T06:58:45.342 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: kBZip2Compression supported: 0 2024-07-15T06:58:45.342 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: kZSTDNotFinalCompression supported: 0 2024-07-15T06:58:45.342 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: kLZ4Compression supported: 1 2024-07-15T06:58:45.343 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: kZlibCompression supported: 1 2024-07-15T06:58:45.343 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: kLZ4HCCompression supported: 1 2024-07-15T06:58:45.343 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: kSnappyCompression supported: 1 2024-07-15T06:58:45.343 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Fast CRC32 supported: Supported on x86 2024-07-15T06:58:45.343 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: DMutex implementation: pthread_mutex_t 2024-07-15T06:58:45.343 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 2024-07-15T06:58:45.343 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-07-15T06:58:45.343 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-07-15T06:58:45.343 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.merge_operator: 2024-07-15T06:58:45.343 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.compaction_filter: None 2024-07-15T06:58:45.343 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.compaction_filter_factory: None 2024-07-15T06:58:45.343 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.sst_partitioner_factory: None 2024-07-15T06:58:45.343 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.memtable_factory: SkipListFactory 2024-07-15T06:58:45.344 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.table_factory: BlockBasedTable 2024-07-15T06:58:45.344 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55c17f52e820) 2024-07-15T06:58:45.344 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: cache_index_and_filter_blocks: 1 2024-07-15T06:58:45.344 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: cache_index_and_filter_blocks_with_high_priority: 0 2024-07-15T06:58:45.344 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: pin_l0_filter_and_index_blocks_in_cache: 0 2024-07-15T06:58:45.344 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: pin_top_level_index_and_filter: 1 2024-07-15T06:58:45.344 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: index_type: 0 2024-07-15T06:58:45.344 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: data_block_index_type: 0 2024-07-15T06:58:45.344 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: index_shortening: 1 2024-07-15T06:58:45.344 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: data_block_hash_table_util_ratio: 0.750000 2024-07-15T06:58:45.344 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: checksum: 4 2024-07-15T06:58:45.344 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: no_block_cache: 0 2024-07-15T06:58:45.344 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: block_cache: 0x55c17f523350 2024-07-15T06:58:45.344 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: block_cache_name: BinnedLRUCache 2024-07-15T06:58:45.345 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: block_cache_options: 2024-07-15T06:58:45.345 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: capacity : 536870912 2024-07-15T06:58:45.345 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: num_shard_bits : 4 2024-07-15T06:58:45.345 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: strict_capacity_limit : 0 2024-07-15T06:58:45.345 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: high_pri_pool_ratio: 0.000 2024-07-15T06:58:45.345 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: block_cache_compressed: (nil) 2024-07-15T06:58:45.345 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: persistent_cache: (nil) 2024-07-15T06:58:45.345 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: block_size: 4096 2024-07-15T06:58:45.345 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: block_size_deviation: 10 2024-07-15T06:58:45.345 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: block_restart_interval: 16 2024-07-15T06:58:45.345 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: index_block_restart_interval: 1 2024-07-15T06:58:45.345 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: metadata_block_size: 4096 2024-07-15T06:58:45.345 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: partition_filters: 0 2024-07-15T06:58:45.345 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: use_delta_encoding: 1 2024-07-15T06:58:45.346 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: filter_policy: bloomfilter 2024-07-15T06:58:45.346 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: whole_key_filtering: 1 2024-07-15T06:58:45.346 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: verify_compression: 0 2024-07-15T06:58:45.346 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: read_amp_bytes_per_bit: 0 2024-07-15T06:58:45.346 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: format_version: 5 2024-07-15T06:58:45.346 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: enable_index_compression: 1 2024-07-15T06:58:45.346 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: block_align: 0 2024-07-15T06:58:45.346 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: max_auto_readahead_size: 262144 2024-07-15T06:58:45.346 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: prepopulate_block_cache: 0 2024-07-15T06:58:45.346 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: initial_auto_readahead_size: 8192 2024-07-15T06:58:45.346 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: num_file_reads_for_auto_readahead: 2 2024-07-15T06:58:45.346 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.write_buffer_size: 33554432 2024-07-15T06:58:45.346 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.max_write_buffer_number: 2 2024-07-15T06:58:45.347 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.compression: NoCompression 2024-07-15T06:58:45.347 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.bottommost_compression: Disabled 2024-07-15T06:58:45.347 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.prefix_extractor: nullptr 2024-07-15T06:58:45.347 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-07-15T06:58:45.347 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.num_levels: 7 2024-07-15T06:58:45.347 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-07-15T06:58:45.347 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-07-15T06:58:45.347 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-07-15T06:58:45.347 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-07-15T06:58:45.347 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2024-07-15T06:58:45.347 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-07-15T06:58:45.347 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-07-15T06:58:45.347 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-07-15T06:58:45.348 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-07-15T06:58:45.348 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2024-07-15T06:58:45.348 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-07-15T06:58:45.348 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-07-15T06:58:45.348 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.compression_opts.window_bits: -14 2024-07-15T06:58:45.348 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.compression_opts.level: 32767 2024-07-15T06:58:45.348 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.compression_opts.strategy: 0 2024-07-15T06:58:45.348 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-07-15T06:58:45.348 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-07-15T06:58:45.348 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-07-15T06:58:45.348 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.compression_opts.parallel_threads: 1 2024-07-15T06:58:45.348 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.compression_opts.enabled: false 2024-07-15T06:58:45.348 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-07-15T06:58:45.349 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-07-15T06:58:45.349 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-07-15T06:58:45.349 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.level0_stop_writes_trigger: 36 2024-07-15T06:58:45.349 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.target_file_size_base: 67108864 2024-07-15T06:58:45.349 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.target_file_size_multiplier: 1 2024-07-15T06:58:45.349 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2024-07-15T06:58:45.349 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-07-15T06:58:45.349 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-07-15T06:58:45.349 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-07-15T06:58:45.349 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-07-15T06:58:45.349 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-07-15T06:58:45.349 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-07-15T06:58:45.349 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-07-15T06:58:45.350 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-07-15T06:58:45.350 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-07-15T06:58:45.350 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-07-15T06:58:45.350 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.max_compaction_bytes: 1677721600 2024-07-15T06:58:45.350 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-07-15T06:58:45.350 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.arena_block_size: 1048576 2024-07-15T06:58:45.350 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-07-15T06:58:45.350 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-07-15T06:58:45.350 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.disable_auto_compactions: 0 2024-07-15T06:58:45.350 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-07-15T06:58:45.350 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-07-15T06:58:45.350 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-07-15T06:58:45.350 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-07-15T06:58:45.351 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-07-15T06:58:45.351 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-07-15T06:58:45.351 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-07-15T06:58:45.351 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-07-15T06:58:45.351 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-07-15T06:58:45.351 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-07-15T06:58:45.351 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.table_properties_collectors: 2024-07-15T06:58:45.351 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.inplace_update_support: 0 2024-07-15T06:58:45.351 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.inplace_update_num_locks: 10000 2024-07-15T06:58:45.351 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-07-15T06:58:45.351 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.memtable_whole_key_filtering: 0 2024-07-15T06:58:45.351 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.memtable_huge_page_size: 0 2024-07-15T06:58:45.351 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.bloom_locality: 0 2024-07-15T06:58:45.351 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.max_successive_merges: 0 2024-07-15T06:58:45.352 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.optimize_filters_for_hits: 0 2024-07-15T06:58:45.352 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.paranoid_file_checks: 0 2024-07-15T06:58:45.352 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.force_consistency_checks: 1 2024-07-15T06:58:45.352 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.report_bg_io_stats: 0 2024-07-15T06:58:45.352 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.ttl: 2592000 2024-07-15T06:58:45.352 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.periodic_compaction_seconds: 0 2024-07-15T06:58:45.352 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.preclude_last_level_data_seconds: 0 2024-07-15T06:58:45.352 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.preserve_internal_time_seconds: 0 2024-07-15T06:58:45.352 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.enable_blob_files: false 2024-07-15T06:58:45.352 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.min_blob_size: 0 2024-07-15T06:58:45.352 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.blob_file_size: 268435456 2024-07-15T06:58:45.352 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.blob_compression_type: NoCompression 2024-07-15T06:58:45.352 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.enable_blob_garbage_collection: false 2024-07-15T06:58:45.353 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-07-15T06:58:45.353 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-07-15T06:58:45.353 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.blob_compaction_readahead_size: 0 2024-07-15T06:58:45.353 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.blob_file_starting_level: 0 2024-07-15T06:58:45.353 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-07-15T06:58:45.353 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f34042640 3 rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 36.sst 2024-07-15T06:58:45.353 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-07-15T06:58:45.353 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 38, last_sequence is 18002, log_number is 34,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-07-15T06:58:45.353 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 34 2024-07-15T06:58:45.353 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 0919f36f-af36-4d21-9a71-e516f4d1bb7b 2024-07-15T06:58:45.353 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1721026725127147, "job": 1, "event": "recovery_started", "wal_files": [34]} 2024-07-15T06:58:45.353 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.120+0000 7f4f3826bb00 4 rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #34 mode 2 2024-07-15T06:58:45.353 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.136+0000 7f4f3826bb00 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1721026725140285, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 39, "file_size": 2593071, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 18008, "largest_seqno": 19512, "table_properties": {"data_size": 2586363, "index_size": 4217, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1477, "raw_key_size": 13358, "raw_average_key_size": 22, "raw_value_size": 2573424, "raw_average_value_size": 4376, "num_data_blocks": 192, "num_entries": 588, "num_filter_entries": 588, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1721026725, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "0919f36f-af36-4d21-9a71-e516f4d1bb7b", "db_session_id": "EFDU2G3TA1P5N2J4L7T6", "orig_file_number": 39, "seqno_to_time_mapping": "N/A"}} 2024-07-15T06:58:45.353 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.136+0000 7f4f3826bb00 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1721026725140425, "job": 1, "event": "recovery_finished"} 2024-07-15T06:58:45.354 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.136+0000 7f4f3826bb00 4 rocksdb: [db/version_set.cc:5047] Creating manifest 41 2024-07-15T06:58:45.354 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.136+0000 7f4f3826bb00 4 rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-07-15T06:58:45.354 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.136+0000 7f4f3826bb00 4 rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000034.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-07-15T06:58:45.354 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.136+0000 7f4f3826bb00 4 rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55c17f54ee00 2024-07-15T06:58:45.354 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.136+0000 7f4f3826bb00 4 rocksdb: DB pointer 0x55c17f646000 2024-07-15T06:58:45.354 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.136+0000 7f4f2e036640 4 rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-07-15T06:58:45.354 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.136+0000 7f4f2e036640 4 rocksdb: [db/db_impl/db_impl.cc:1111] 2024-07-15T06:58:45.354 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: ** DB Stats ** 2024-07-15T06:58:45.354 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: Uptime(secs): 0.0 total, 0.0 interval 2024-07-15T06:58:45.354 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-07-15T06:58:45.354 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-07-15T06:58:45.354 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-07-15T06:58:45.354 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-07-15T06:58:45.355 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-07-15T06:58:45.355 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-07-15T06:58:45.355 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: ** Compaction Stats [default] ** 2024-07-15T06:58:45.355 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-07-15T06:58:45.355 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-07-15T06:58:45.355 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: L0 1/0 2.47 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 544.9 0.00 0.00 1 0.005 0 0 0.0 0.0 2024-07-15T06:58:45.355 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: L6 1/0 7.17 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-07-15T06:58:45.355 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: Sum 2/0 9.64 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 544.9 0.00 0.00 1 0.005 0 0 0.0 0.0 2024-07-15T06:58:45.355 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 544.9 0.00 0.00 1 0.005 0 0 0.0 0.0 2024-07-15T06:58:45.355 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: ** Compaction Stats [default] ** 2024-07-15T06:58:45.355 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-07-15T06:58:45.355 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-07-15T06:58:45.355 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 544.9 0.00 0.00 1 0.005 0 0 0.0 0.0 2024-07-15T06:58:45.355 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-07-15T06:58:45.356 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: Uptime(secs): 0.0 total, 0.0 interval 2024-07-15T06:58:45.356 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: Flush(GB): cumulative 0.002, interval 0.002 2024-07-15T06:58:45.356 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: AddFile(GB): cumulative 0.000, interval 0.000 2024-07-15T06:58:45.356 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: AddFile(Total Files): cumulative 0, interval 0 2024-07-15T06:58:45.356 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: AddFile(L0 Files): cumulative 0, interval 0 2024-07-15T06:58:45.356 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: AddFile(Keys): cumulative 0, interval 0 2024-07-15T06:58:45.356 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: Cumulative compaction: 0.00 GB write, 152.43 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-07-15T06:58:45.356 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: Interval compaction: 0.00 GB write, 152.43 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-07-15T06:58:45.356 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-07-15T06:58:45.356 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: Block cache BinnedLRUCache@0x55c17f523350#7 capacity: 512.00 MB usage: 6.11 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.7e-05 secs_since: 0 2024-07-15T06:58:45.356 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: Block cache entry stats(count,size,portion): FilterBlock(1,1.53 KB,0.000292063%) IndexBlock(1,4.58 KB,0.000873208%) Misc(1,0.00 KB,0%) 2024-07-15T06:58:45.356 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: ** File Read Latency Histogram By Level [default] ** 2024-07-15T06:58:45.356 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.136+0000 7f4f3826bb00 0 starting mon.c rank 2 at public addrs [v2:172.21.15.89:3301/0,v1:172.21.15.89:6790/0] at bind addrs [v2:172.21.15.89:3301/0,v1:172.21.15.89:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid 75ba02bc-4274-11ef-bcad-c7b262605968 2024-07-15T06:58:45.357 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.140+0000 7f4f3826bb00 1 mon.c@-1(???) e3 preinit fsid 75ba02bc-4274-11ef-bcad-c7b262605968 2024-07-15T06:58:45.357 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.140+0000 7f4f3826bb00 0 mon.c@-1(???).mds e10 new map 2024-07-15T06:58:45.357 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.140+0000 7f4f3826bb00 0 mon.c@-1(???).mds e10 print_map 2024-07-15T06:58:45.357 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: e10 2024-07-15T06:58:45.357 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: enable_multiple, ever_enabled_multiple: 1,1 2024-07-15T06:58:45.357 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-07-15T06:58:45.357 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: legacy client fscid: 1 2024-07-15T06:58:45.357 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: 2024-07-15T06:58:45.357 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: Filesystem 'foo' (1) 2024-07-15T06:58:45.357 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: fs_name foo 2024-07-15T06:58:45.357 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: epoch 10 2024-07-15T06:58:45.357 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: flags 12 joinable allow_snaps allow_multimds_snaps 2024-07-15T06:58:45.358 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: created 2024-07-15T06:50:29.283802+0000 2024-07-15T06:58:45.358 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: modified 2024-07-15T06:50:39.720043+0000 2024-07-15T06:58:45.358 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: tableserver 0 2024-07-15T06:58:45.358 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: root 0 2024-07-15T06:58:45.358 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: session_timeout 60 2024-07-15T06:58:45.358 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: session_autoclose 300 2024-07-15T06:58:45.358 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: max_file_size 1099511627776 2024-07-15T06:58:45.358 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: max_xattr_size 65536 2024-07-15T06:58:45.358 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: required_client_features {} 2024-07-15T06:58:45.358 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: last_failure 0 2024-07-15T06:58:45.358 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: last_failure_osd_epoch 68 2024-07-15T06:58:45.358 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-07-15T06:58:45.358 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: max_mds 1 2024-07-15T06:58:45.359 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: in 0 2024-07-15T06:58:45.359 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: up {0=24415} 2024-07-15T06:58:45.359 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: failed 2024-07-15T06:58:45.359 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: damaged 2024-07-15T06:58:45.359 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: stopped 2024-07-15T06:58:45.359 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: data_pools [4] 2024-07-15T06:58:45.359 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: metadata_pool 3 2024-07-15T06:58:45.359 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: inline_data disabled 2024-07-15T06:58:45.359 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: balancer 2024-07-15T06:58:45.359 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: bal_rank_mask -1 2024-07-15T06:58:45.359 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: standby_count_wanted 1 2024-07-15T06:58:45.359 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: [mds.foo.smithi134.wugtzh{0:24415} state up:active seq 4 join_fscid=1 addr [v2:172.21.15.134:6832/968641534,v1:172.21.15.134:6833/968641534] compat {c=[1],r=[1],i=[7ff]}] 2024-07-15T06:58:45.360 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: 2024-07-15T06:58:45.360 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: 2024-07-15T06:58:45.360 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: Standby daemons: 2024-07-15T06:58:45.360 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: 2024-07-15T06:58:45.360 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: [mds.foo.smithi089.cczuoz{-1:24488} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.89:6834/1598068998,v1:172.21.15.89:6835/1598068998] compat {c=[1],r=[1],i=[7ff]}] 2024-07-15T06:58:45.360 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.140+0000 7f4f3826bb00 0 mon.c@-1(???).osd e79 crush map has features 3314933000854323200, adjusting msgr requires 2024-07-15T06:58:45.360 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.140+0000 7f4f3826bb00 0 mon.c@-1(???).osd e79 crush map has features 432629239337189376, adjusting msgr requires 2024-07-15T06:58:45.360 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.140+0000 7f4f3826bb00 0 mon.c@-1(???).osd e79 crush map has features 432629239337189376, adjusting msgr requires 2024-07-15T06:58:45.360 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.140+0000 7f4f3826bb00 0 mon.c@-1(???).osd e79 crush map has features 432629239337189376, adjusting msgr requires 2024-07-15T06:58:45.360 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:45 smithi089 bash[82202]: debug 2024-07-15T06:58:45.140+0000 7f4f3826bb00 1 mon.c@-1(???).paxosservice(auth 1..22) refresh upgraded, format 0 -> 3 2024-07-15T06:58:47.353 INFO:teuthology.orchestra.run.smithi089.stdout:wait for majority of mons to upgrade 2024-07-15T06:58:48.115 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -e sha1=6ea7aaa9f93d34dc4d21862afe00140dfddbd706 -- bash -c 'while ! ceph mon versions | grep $sha1 | egrep '"'"': [23]'"'"' ; do sleep 2 ; done' 2024-07-15T06:58:49.054 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:58:48 smithi134 bash[60729]: ts=2024-07-15T06:58:48.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1102-g9a202024 (9a2020246f85463d4fdb115c94aa40b463caa1d2) quincy (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1102-g9a202024 (9a2020246f85463d4fdb115c94aa40b463caa1d2) quincy (stable)\", cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:58:50.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:50 smithi089 bash[79810]: audit 2024-07-15T06:58:41.611395+0000 mon.a (mon.0) 52 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-07-15T06:58:50.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:50 smithi089 bash[79810]: audit 2024-07-15T06:58:41.611805+0000 mon.a (mon.0) 53 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-15T06:58:50.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:50 smithi089 bash[79810]: audit 2024-07-15T06:58:41.611995+0000 mon.a (mon.0) 54 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-15T06:58:50.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:50 smithi089 bash[79810]: debug 2024-07-15T06:58:50.400+0000 7fb55ea00640 -1 log_channel(cluster) log [ERR] : Health detail: HEALTH_ERR 1/3 mons down, quorum a,b; 8 osds(s) are not reachable 2024-07-15T06:58:50.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:50 smithi089 bash[79810]: debug 2024-07-15T06:58:50.400+0000 7fb55ea00640 -1 log_channel(cluster) log [ERR] : [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2024-07-15T06:58:50.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:50 smithi089 bash[79810]: debug 2024-07-15T06:58:50.400+0000 7fb55ea00640 -1 log_channel(cluster) log [ERR] : mon.c (rank 2) addr [v2:172.21.15.89:3301/0,v1:172.21.15.89:6790/0] is down (out of quorum) 2024-07-15T06:58:50.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:50 smithi089 bash[79810]: debug 2024-07-15T06:58:50.400+0000 7fb55ea00640 -1 log_channel(cluster) log [ERR] : [ERR] OSD_UNREACHABLE: 8 osds(s) are not reachable 2024-07-15T06:58:50.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:50 smithi089 bash[79810]: debug 2024-07-15T06:58:50.400+0000 7fb55ea00640 -1 log_channel(cluster) log [ERR] : osd.0's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:58:50.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:50 smithi089 bash[79810]: debug 2024-07-15T06:58:50.400+0000 7fb55ea00640 -1 log_channel(cluster) log [ERR] : osd.1's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:58:50.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:50 smithi089 bash[79810]: debug 2024-07-15T06:58:50.400+0000 7fb55ea00640 -1 log_channel(cluster) log [ERR] : osd.2's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:58:50.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:50 smithi089 bash[79810]: debug 2024-07-15T06:58:50.400+0000 7fb55ea00640 -1 log_channel(cluster) log [ERR] : osd.3's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:58:50.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:50 smithi089 bash[79810]: debug 2024-07-15T06:58:50.400+0000 7fb55ea00640 -1 log_channel(cluster) log [ERR] : osd.4's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:58:50.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:50 smithi089 bash[79810]: debug 2024-07-15T06:58:50.400+0000 7fb55ea00640 -1 log_channel(cluster) log [ERR] : osd.5's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:58:50.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:50 smithi089 bash[79810]: debug 2024-07-15T06:58:50.400+0000 7fb55ea00640 -1 log_channel(cluster) log [ERR] : osd.6's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:58:50.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:50 smithi089 bash[79810]: debug 2024-07-15T06:58:50.400+0000 7fb55ea00640 -1 log_channel(cluster) log [ERR] : osd.7's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:58:50.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:58:50 smithi134 bash[24068]: audit 2024-07-15T06:58:41.611395+0000 mon.a (mon.0) 52 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-07-15T06:58:50.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:58:50 smithi134 bash[24068]: audit 2024-07-15T06:58:41.611805+0000 mon.a (mon.0) 53 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-15T06:58:50.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:58:50 smithi134 bash[24068]: audit 2024-07-15T06:58:41.611995+0000 mon.a (mon.0) 54 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-15T06:58:51.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:58:51 smithi134 bash[24068]: cluster 2024-07-15T06:58:45.357084+0000 mon.c (mon.2) 1 : cluster [INF] mon.c calling monitor election 2024-07-15T06:58:51.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:58:51 smithi134 bash[24068]: cluster 2024-07-15T06:58:51.364011+0000 mon.b (mon.1) 139 : cluster [INF] mon.b calling monitor election 2024-07-15T06:58:51.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:58:51 smithi134 bash[24068]: cluster 2024-07-15T06:58:51.364037+0000 mon.a (mon.0) 78 : cluster [INF] mon.a calling monitor election 2024-07-15T06:58:51.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:58:51 smithi134 bash[24068]: cluster 2024-07-15T06:58:51.387460+0000 mon.a (mon.0) 79 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-07-15T06:58:51.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:58:51 smithi134 bash[24068]: cluster 2024-07-15T06:58:51.398312+0000 mon.a (mon.0) 80 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.89:3300/0,v1:172.21.15.89:6789/0],b=[v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0],c=[v2:172.21.15.89:3301/0,v1:172.21.15.89:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-07-15T06:58:51.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:58:51 smithi134 bash[24068]: cluster 2024-07-15T06:58:51.400024+0000 mon.a (mon.0) 81 : cluster [DBG] fsmap foo:1 {0=foo.smithi134.wugtzh=up:active} 1 up:standby 2024-07-15T06:58:51.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:58:51 smithi134 bash[24068]: cluster 2024-07-15T06:58:51.400120+0000 mon.a (mon.0) 82 : cluster [DBG] osdmap e79: 8 total, 8 up, 8 in 2024-07-15T06:58:51.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:58:51 smithi134 bash[24068]: cluster 2024-07-15T06:58:51.401100+0000 mon.a (mon.0) 83 : cluster [DBG] mgrmap e40: y(active, since 106s), standbys: x 2024-07-15T06:58:51.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:58:51 smithi134 bash[24068]: cluster 2024-07-15T06:58:51.401604+0000 mon.a (mon.0) 84 : cluster [INF] Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-07-15T06:58:51.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:58:51 smithi134 bash[24068]: cluster 2024-07-15T06:58:51.416309+0000 mon.a (mon.0) 85 : cluster [ERR] Health detail: HEALTH_ERR 8 osds(s) are not reachable 2024-07-15T06:58:51.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:58:51 smithi134 bash[24068]: cluster 2024-07-15T06:58:51.416360+0000 mon.a (mon.0) 86 : cluster [ERR] [ERR] OSD_UNREACHABLE: 8 osds(s) are not reachable 2024-07-15T06:58:51.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:58:51 smithi134 bash[24068]: cluster 2024-07-15T06:58:51.416396+0000 mon.a (mon.0) 87 : cluster [ERR] osd.0's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:58:51.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:58:51 smithi134 bash[24068]: cluster 2024-07-15T06:58:51.416436+0000 mon.a (mon.0) 88 : cluster [ERR] osd.1's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:58:51.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:58:51 smithi134 bash[24068]: cluster 2024-07-15T06:58:51.416469+0000 mon.a (mon.0) 89 : cluster [ERR] osd.2's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:58:51.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:58:51 smithi134 bash[24068]: cluster 2024-07-15T06:58:51.416544+0000 mon.a (mon.0) 90 : cluster [ERR] osd.3's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:58:51.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:58:51 smithi134 bash[24068]: cluster 2024-07-15T06:58:51.416586+0000 mon.a (mon.0) 91 : cluster [ERR] osd.4's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:58:51.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:58:51 smithi134 bash[24068]: cluster 2024-07-15T06:58:51.416624+0000 mon.a (mon.0) 92 : cluster [ERR] osd.5's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:58:51.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:58:51 smithi134 bash[24068]: cluster 2024-07-15T06:58:51.416658+0000 mon.a (mon.0) 93 : cluster [ERR] osd.6's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:58:51.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:58:51 smithi134 bash[24068]: cluster 2024-07-15T06:58:51.416687+0000 mon.a (mon.0) 94 : cluster [ERR] osd.7's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:58:51.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:51 smithi089 bash[79810]: debug 2024-07-15T06:58:51.412+0000 7fb55ea00640 -1 log_channel(cluster) log [ERR] : Health detail: HEALTH_ERR 8 osds(s) are not reachable 2024-07-15T06:58:51.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:51 smithi089 bash[79810]: debug 2024-07-15T06:58:51.412+0000 7fb55ea00640 -1 log_channel(cluster) log [ERR] : [ERR] OSD_UNREACHABLE: 8 osds(s) are not reachable 2024-07-15T06:58:51.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:51 smithi089 bash[79810]: debug 2024-07-15T06:58:51.412+0000 7fb55ea00640 -1 log_channel(cluster) log [ERR] : osd.0's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:58:51.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:51 smithi089 bash[79810]: debug 2024-07-15T06:58:51.412+0000 7fb55ea00640 -1 log_channel(cluster) log [ERR] : osd.1's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:58:51.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:51 smithi089 bash[79810]: debug 2024-07-15T06:58:51.412+0000 7fb55ea00640 -1 log_channel(cluster) log [ERR] : osd.2's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:58:51.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:51 smithi089 bash[79810]: debug 2024-07-15T06:58:51.412+0000 7fb55ea00640 -1 log_channel(cluster) log [ERR] : osd.3's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:58:51.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:51 smithi089 bash[79810]: debug 2024-07-15T06:58:51.412+0000 7fb55ea00640 -1 log_channel(cluster) log [ERR] : osd.4's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:58:51.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:51 smithi089 bash[79810]: debug 2024-07-15T06:58:51.412+0000 7fb55ea00640 -1 log_channel(cluster) log [ERR] : osd.5's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:58:51.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:51 smithi089 bash[79810]: debug 2024-07-15T06:58:51.412+0000 7fb55ea00640 -1 log_channel(cluster) log [ERR] : osd.6's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:58:51.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:51 smithi089 bash[79810]: debug 2024-07-15T06:58:51.412+0000 7fb55ea00640 -1 log_channel(cluster) log [ERR] : osd.7's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:58:51.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:51 smithi089 bash[79810]: cluster 2024-07-15T06:58:45.357084+0000 mon.c (mon.2) 1 : cluster [INF] mon.c calling monitor election 2024-07-15T06:58:51.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:51 smithi089 bash[79810]: cluster 2024-07-15T06:58:51.364011+0000 mon.b (mon.1) 139 : cluster [INF] mon.b calling monitor election 2024-07-15T06:58:51.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:51 smithi089 bash[79810]: cluster 2024-07-15T06:58:51.364037+0000 mon.a (mon.0) 78 : cluster [INF] mon.a calling monitor election 2024-07-15T06:58:51.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:51 smithi089 bash[79810]: cluster 2024-07-15T06:58:51.387460+0000 mon.a (mon.0) 79 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-07-15T06:58:51.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:51 smithi089 bash[79810]: cluster 2024-07-15T06:58:51.398312+0000 mon.a (mon.0) 80 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.89:3300/0,v1:172.21.15.89:6789/0],b=[v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0],c=[v2:172.21.15.89:3301/0,v1:172.21.15.89:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-07-15T06:58:51.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:51 smithi089 bash[79810]: cluster 2024-07-15T06:58:51.400024+0000 mon.a (mon.0) 81 : cluster [DBG] fsmap foo:1 {0=foo.smithi134.wugtzh=up:active} 1 up:standby 2024-07-15T06:58:51.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:51 smithi089 bash[79810]: cluster 2024-07-15T06:58:51.400120+0000 mon.a (mon.0) 82 : cluster [DBG] osdmap e79: 8 total, 8 up, 8 in 2024-07-15T06:58:51.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:51 smithi089 bash[79810]: cluster 2024-07-15T06:58:51.401100+0000 mon.a (mon.0) 83 : cluster [DBG] mgrmap e40: y(active, since 106s), standbys: x 2024-07-15T06:58:51.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:51 smithi089 bash[79810]: cluster 2024-07-15T06:58:51.401604+0000 mon.a (mon.0) 84 : cluster [INF] Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-07-15T06:58:51.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:51 smithi089 bash[79810]: cluster 2024-07-15T06:58:51.416309+0000 mon.a (mon.0) 85 : cluster [ERR] Health detail: HEALTH_ERR 8 osds(s) are not reachable 2024-07-15T06:58:51.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:51 smithi089 bash[79810]: cluster 2024-07-15T06:58:51.416360+0000 mon.a (mon.0) 86 : cluster [ERR] [ERR] OSD_UNREACHABLE: 8 osds(s) are not reachable 2024-07-15T06:58:51.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:51 smithi089 bash[82202]: audit 2024-07-15T06:58:41.611395+0000 mon.a (mon.0) 52 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-07-15T06:58:51.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:51 smithi089 bash[82202]: audit 2024-07-15T06:58:41.611805+0000 mon.a (mon.0) 53 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-15T06:58:51.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:51 smithi089 bash[82202]: audit 2024-07-15T06:58:41.611995+0000 mon.a (mon.0) 54 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-15T06:58:51.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:51 smithi089 bash[82202]: cluster 2024-07-15T06:58:45.357084+0000 mon.c (mon.2) 1 : cluster [INF] mon.c calling monitor election 2024-07-15T06:58:51.873 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:51 smithi089 bash[82202]: cluster 2024-07-15T06:58:51.364011+0000 mon.b (mon.1) 139 : cluster [INF] mon.b calling monitor election 2024-07-15T06:58:51.873 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:51 smithi089 bash[82202]: cluster 2024-07-15T06:58:51.364037+0000 mon.a (mon.0) 78 : cluster [INF] mon.a calling monitor election 2024-07-15T06:58:51.873 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:51 smithi089 bash[82202]: cluster 2024-07-15T06:58:51.387460+0000 mon.a (mon.0) 79 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-07-15T06:58:51.873 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:51 smithi089 bash[82202]: cluster 2024-07-15T06:58:51.398312+0000 mon.a (mon.0) 80 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.89:3300/0,v1:172.21.15.89:6789/0],b=[v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0],c=[v2:172.21.15.89:3301/0,v1:172.21.15.89:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-07-15T06:58:51.873 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:51 smithi089 bash[82202]: cluster 2024-07-15T06:58:51.400024+0000 mon.a (mon.0) 81 : cluster [DBG] fsmap foo:1 {0=foo.smithi134.wugtzh=up:active} 1 up:standby 2024-07-15T06:58:51.873 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:51 smithi089 bash[82202]: cluster 2024-07-15T06:58:51.400120+0000 mon.a (mon.0) 82 : cluster [DBG] osdmap e79: 8 total, 8 up, 8 in 2024-07-15T06:58:51.873 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:51 smithi089 bash[82202]: cluster 2024-07-15T06:58:51.401100+0000 mon.a (mon.0) 83 : cluster [DBG] mgrmap e40: y(active, since 106s), standbys: x 2024-07-15T06:58:51.873 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:51 smithi089 bash[82202]: cluster 2024-07-15T06:58:51.401604+0000 mon.a (mon.0) 84 : cluster [INF] Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-07-15T06:58:51.873 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:51 smithi089 bash[82202]: cluster 2024-07-15T06:58:51.416309+0000 mon.a (mon.0) 85 : cluster [ERR] Health detail: HEALTH_ERR 8 osds(s) are not reachable 2024-07-15T06:58:51.873 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:51 smithi089 bash[82202]: cluster 2024-07-15T06:58:51.416360+0000 mon.a (mon.0) 86 : cluster [ERR] [ERR] OSD_UNREACHABLE: 8 osds(s) are not reachable 2024-07-15T06:58:51.873 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:51 smithi089 bash[82202]: cluster 2024-07-15T06:58:51.416396+0000 mon.a (mon.0) 87 : cluster [ERR] osd.0's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:58:51.873 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:51 smithi089 bash[82202]: cluster 2024-07-15T06:58:51.416436+0000 mon.a (mon.0) 88 : cluster [ERR] osd.1's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:58:51.873 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:51 smithi089 bash[82202]: cluster 2024-07-15T06:58:51.416469+0000 mon.a (mon.0) 89 : cluster [ERR] osd.2's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:58:51.874 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:51 smithi089 bash[82202]: cluster 2024-07-15T06:58:51.416544+0000 mon.a (mon.0) 90 : cluster [ERR] osd.3's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:58:51.874 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:51 smithi089 bash[82202]: cluster 2024-07-15T06:58:51.416586+0000 mon.a (mon.0) 91 : cluster [ERR] osd.4's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:58:51.874 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:51 smithi089 bash[82202]: cluster 2024-07-15T06:58:51.416624+0000 mon.a (mon.0) 92 : cluster [ERR] osd.5's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:58:51.874 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:51 smithi089 bash[82202]: cluster 2024-07-15T06:58:51.416658+0000 mon.a (mon.0) 93 : cluster [ERR] osd.6's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:58:51.874 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:51 smithi089 bash[82202]: cluster 2024-07-15T06:58:51.416687+0000 mon.a (mon.0) 94 : cluster [ERR] osd.7's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:58:51.874 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:51 smithi089 bash[79810]: cluster 2024-07-15T06:58:51.416396+0000 mon.a (mon.0) 87 : cluster [ERR] osd.0's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:58:51.874 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:51 smithi089 bash[79810]: cluster 2024-07-15T06:58:51.416436+0000 mon.a (mon.0) 88 : cluster [ERR] osd.1's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:58:51.874 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:51 smithi089 bash[79810]: cluster 2024-07-15T06:58:51.416469+0000 mon.a (mon.0) 89 : cluster [ERR] osd.2's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:58:51.874 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:51 smithi089 bash[79810]: cluster 2024-07-15T06:58:51.416544+0000 mon.a (mon.0) 90 : cluster [ERR] osd.3's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:58:51.874 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:51 smithi089 bash[79810]: cluster 2024-07-15T06:58:51.416586+0000 mon.a (mon.0) 91 : cluster [ERR] osd.4's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:58:51.875 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:51 smithi089 bash[79810]: cluster 2024-07-15T06:58:51.416624+0000 mon.a (mon.0) 92 : cluster [ERR] osd.5's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:58:51.875 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:51 smithi089 bash[79810]: cluster 2024-07-15T06:58:51.416658+0000 mon.a (mon.0) 93 : cluster [ERR] osd.6's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:58:51.875 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:51 smithi089 bash[79810]: cluster 2024-07-15T06:58:51.416687+0000 mon.a (mon.0) 94 : cluster [ERR] osd.7's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:58:52.804 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:58:52 smithi134 bash[60729]: ts=2024-07-15T06:58:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:58:52.889 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T06:58:53.792 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:58:53 smithi134 bash[60729]: ts=2024-07-15T06:58:53.508Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi134\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi134\", job=\"node\", machine=\"x86_64\", nodename=\"smithi134\", release=\"5.15.0-113-generic\", sysname=\"Linux\", version=\"#123-Ubuntu SMP Mon Jun 10 08:16:17 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", domainname=\"(none)\", instance=\"smithi134\", job=\"node\", machine=\"x86_64\", nodename=\"smithi134\", release=\"5.15.0-113-generic\", sysname=\"Linux\", version=\"#123-Ubuntu SMP Mon Jun 10 08:16:17 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:58:54.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:58:53 smithi134 bash[24068]: cluster 2024-07-15T06:58:52.740467+0000 mgr.y (mgr.25244) 76 : cluster [DBG] pgmap v57: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:58:54.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:53 smithi089 bash[79810]: cluster 2024-07-15T06:58:52.740467+0000 mgr.y (mgr.25244) 76 : cluster [DBG] pgmap v57: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:58:54.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:53 smithi089 bash[82202]: cluster 2024-07-15T06:58:52.740467+0000 mgr.y (mgr.25244) 76 : cluster [DBG] pgmap v57: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:58:54.530 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:58:54 smithi089 bash[72894]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:58:54] "GET /metrics HTTP/1.1" 200 33857 "" "Prometheus/2.43.0" 2024-07-15T06:58:54.804 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:58:54 smithi134 bash[60729]: ts=2024-07-15T06:58:54.454Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:58:54.804 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:58:54 smithi134 bash[60729]: ts=2024-07-15T06:58:54.455Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:58:55.622 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:58:55 smithi089 bash[72894]: debug 2024-07-15T06:58:55.148+0000 7f112575c640 -1 mgr.server handle_report got status from non-daemon mon.c 2024-07-15T06:58:56.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:58:55 smithi134 bash[24068]: cluster 2024-07-15T06:58:54.741183+0000 mgr.y (mgr.25244) 77 : cluster [DBG] pgmap v58: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:58:56.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:55 smithi089 bash[79810]: cluster 2024-07-15T06:58:54.741183+0000 mgr.y (mgr.25244) 77 : cluster [DBG] pgmap v58: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:58:56.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:55 smithi089 bash[82202]: cluster 2024-07-15T06:58:54.741183+0000 mgr.y (mgr.25244) 77 : cluster [DBG] pgmap v58: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:58:56.696 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 18.2.2-1609-g6ea7aaa9 (6ea7aaa9f93d34dc4d21862afe00140dfddbd706) reef (stable)": 2 2024-07-15T06:58:57.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:57 smithi089 bash[79810]: audit 2024-07-15T06:58:56.141637+0000 mon.a (mon.0) 95 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:58:57.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:57 smithi089 bash[79810]: audit 2024-07-15T06:58:56.146890+0000 mon.a (mon.0) 96 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:58:57.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:57 smithi089 bash[79810]: audit 2024-07-15T06:58:56.688779+0000 mon.a (mon.0) 97 : audit [DBG] from='client.? 172.21.15.89:0/2026694854' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:58:57.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:57 smithi089 bash[79810]: cluster 2024-07-15T06:58:56.741742+0000 mgr.y (mgr.25244) 78 : cluster [DBG] pgmap v59: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:58:57.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:57 smithi089 bash[79810]: audit 2024-07-15T06:58:56.782866+0000 mon.a (mon.0) 98 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:58:57.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:57 smithi089 bash[79810]: audit 2024-07-15T06:58:56.788150+0000 mon.a (mon.0) 99 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:58:57.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:57 smithi089 bash[82202]: audit 2024-07-15T06:58:56.141637+0000 mon.a (mon.0) 95 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:58:57.371 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:57 smithi089 bash[82202]: audit 2024-07-15T06:58:56.146890+0000 mon.a (mon.0) 96 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:58:57.371 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:57 smithi089 bash[82202]: audit 2024-07-15T06:58:56.688779+0000 mon.a (mon.0) 97 : audit [DBG] from='client.? 172.21.15.89:0/2026694854' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:58:57.371 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:57 smithi089 bash[82202]: cluster 2024-07-15T06:58:56.741742+0000 mgr.y (mgr.25244) 78 : cluster [DBG] pgmap v59: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:58:57.371 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:57 smithi089 bash[82202]: audit 2024-07-15T06:58:56.782866+0000 mon.a (mon.0) 98 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:58:57.371 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:57 smithi089 bash[82202]: audit 2024-07-15T06:58:56.788150+0000 mon.a (mon.0) 99 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:58:57.519 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -e sha1=6ea7aaa9f93d34dc4d21862afe00140dfddbd706 -- bash -c 'ceph orch ps' 2024-07-15T06:58:57.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:58:57 smithi134 bash[24068]: audit 2024-07-15T06:58:56.141637+0000 mon.a (mon.0) 95 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:58:57.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:58:57 smithi134 bash[24068]: audit 2024-07-15T06:58:56.146890+0000 mon.a (mon.0) 96 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:58:57.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:58:57 smithi134 bash[24068]: audit 2024-07-15T06:58:56.688779+0000 mon.a (mon.0) 97 : audit [DBG] from='client.? 172.21.15.89:0/2026694854' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T06:58:57.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:58:57 smithi134 bash[24068]: cluster 2024-07-15T06:58:56.741742+0000 mgr.y (mgr.25244) 78 : cluster [DBG] pgmap v59: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:58:57.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:58:57 smithi134 bash[24068]: audit 2024-07-15T06:58:56.782866+0000 mon.a (mon.0) 98 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:58:57.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:58:57 smithi134 bash[24068]: audit 2024-07-15T06:58:56.788150+0000 mon.a (mon.0) 99 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:58:59.054 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:58:58 smithi134 bash[60729]: ts=2024-07-15T06:58:58.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1102-g9a202024 (9a2020246f85463d4fdb115c94aa40b463caa1d2) quincy (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1102-g9a202024 (9a2020246f85463d4fdb115c94aa40b463caa1d2) quincy (stable)\", cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:59:00.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:58:59 smithi134 bash[24068]: cluster 2024-07-15T06:58:58.742365+0000 mgr.y (mgr.25244) 79 : cluster [DBG] pgmap v60: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:59:00.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:58:59 smithi089 bash[79810]: cluster 2024-07-15T06:58:58.742365+0000 mgr.y (mgr.25244) 79 : cluster [DBG] pgmap v60: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:59:00.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:58:59 smithi089 bash[82202]: cluster 2024-07-15T06:58:58.742365+0000 mgr.y (mgr.25244) 79 : cluster [DBG] pgmap v60: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:59:00.554 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:59:00 smithi134 bash[60729]: ts=2024-07-15T06:59:00.088Z caller=alerting.go:366 level=warn component="rule manager" alert="unsupported value type" msg="Expanding alert template failed" err="error executing template __alert_CephMonDown: template: __alert_CephMonDown:1:427: executing \"__alert_CephMonDown\" at : error calling query: found duplicate series for the match group {ceph_daemon=\"mon.a\"} on the right hand-side of the operation: [{ceph_daemon=\"mon.a\", ceph_version=\"ceph version 18.2.2-1609-g6ea7aaa9 (6ea7aaa9f93d34dc4d21862afe00140dfddbd706) reef (stable)\", cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", hostname=\"smithi089\", instance=\"ceph_cluster\", job=\"ceph\", public_addr=\"172.21.15.89\", rank=\"0\"}, {ceph_daemon=\"mon.a\", ceph_version=\"ceph version 17.2.7-1102-g9a202024 (9a2020246f85463d4fdb115c94aa40b463caa1d2) quincy (stable)\", hostname=\"smithi089\", instance=\"ceph_cluster\", job=\"ceph\", public_addr=\"172.21.15.89\", rank=\"0\"}];many-to-many matching not allowed: matching labels must be unique on one side" data="unsupported value type" 2024-07-15T06:59:01.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:00 smithi089 bash[79810]: cluster 2024-07-15T06:59:00.742997+0000 mgr.y (mgr.25244) 80 : cluster [DBG] pgmap v61: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:59:01.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:00 smithi089 bash[82202]: cluster 2024-07-15T06:59:00.742997+0000 mgr.y (mgr.25244) 80 : cluster [DBG] pgmap v61: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:59:01.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:00 smithi134 bash[24068]: cluster 2024-07-15T06:59:00.742997+0000 mgr.y (mgr.25244) 80 : cluster [DBG] pgmap v61: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:59:01.619 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T06:59:02.804 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:59:02 smithi134 bash[60729]: ts=2024-07-15T06:59:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:59:03.795 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:59:03 smithi134 bash[60729]: ts=2024-07-15T06:59:03.507Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi134\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi134\", job=\"node\", machine=\"x86_64\", nodename=\"smithi134\", release=\"5.15.0-113-generic\", sysname=\"Linux\", version=\"#123-Ubuntu SMP Mon Jun 10 08:16:17 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", domainname=\"(none)\", instance=\"smithi134\", job=\"node\", machine=\"x86_64\", nodename=\"smithi134\", release=\"5.15.0-113-generic\", sysname=\"Linux\", version=\"#123-Ubuntu SMP Mon Jun 10 08:16:17 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:59:04.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:03 smithi134 bash[24068]: cluster 2024-07-15T06:59:02.744094+0000 mgr.y (mgr.25244) 81 : cluster [DBG] pgmap v62: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:59:04.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:03 smithi089 bash[79810]: cluster 2024-07-15T06:59:02.744094+0000 mgr.y (mgr.25244) 81 : cluster [DBG] pgmap v62: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:59:04.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:03 smithi089 bash[82202]: cluster 2024-07-15T06:59:02.744094+0000 mgr.y (mgr.25244) 81 : cluster [DBG] pgmap v62: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:59:04.533 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:59:04 smithi089 bash[72894]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:59:04] "GET /metrics HTTP/1.1" 200 33857 "" "Prometheus/2.43.0" 2024-07-15T06:59:04.804 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:59:04 smithi134 bash[60729]: ts=2024-07-15T06:59:04.454Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:59:04.804 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:59:04 smithi134 bash[60729]: ts=2024-07-15T06:59:04.455Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:59:05.090 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:04 smithi089 bash[79810]: audit 2024-07-15T06:59:04.786145+0000 mon.a (mon.0) 100 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:59:05.091 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:04 smithi089 bash[82202]: audit 2024-07-15T06:59:04.786145+0000 mon.a (mon.0) 100 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:59:05.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:04 smithi134 bash[24068]: audit 2024-07-15T06:59:04.786145+0000 mon.a (mon.0) 100 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:59:06.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:05 smithi089 bash[79810]: cluster 2024-07-15T06:59:04.744721+0000 mgr.y (mgr.25244) 82 : cluster [DBG] pgmap v63: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:59:06.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:05 smithi089 bash[82202]: cluster 2024-07-15T06:59:04.744721+0000 mgr.y (mgr.25244) 82 : cluster [DBG] pgmap v63: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:59:06.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:05 smithi134 bash[24068]: cluster 2024-07-15T06:59:04.744721+0000 mgr.y (mgr.25244) 82 : cluster [DBG] pgmap v63: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:59:07.080 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:06 smithi089 bash[79810]: cluster 2024-07-15T06:59:06.745888+0000 mgr.y (mgr.25244) 83 : cluster [DBG] pgmap v64: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:59:07.080 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:06 smithi089 bash[82202]: cluster 2024-07-15T06:59:06.745888+0000 mgr.y (mgr.25244) 83 : cluster [DBG] pgmap v64: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:59:07.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:06 smithi134 bash[24068]: cluster 2024-07-15T06:59:06.745888+0000 mgr.y (mgr.25244) 83 : cluster [DBG] pgmap v64: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:59:07.471 INFO:teuthology.orchestra.run.smithi089.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-07-15T06:59:07.471 INFO:teuthology.orchestra.run.smithi089.stdout:alertmanager.a smithi089 *:9093,9094 running (3m) 11s ago 11m 14.0M - 0.25.0 c8568f914cd2 c476af8941dd 2024-07-15T06:59:07.471 INFO:teuthology.orchestra.run.smithi089.stdout:grafana.a smithi134 *:3000 running (3m) 39s ago 10m 75.2M - 9.4.7 954c08fa6188 ac957811e4af 2024-07-15T06:59:07.471 INFO:teuthology.orchestra.run.smithi089.stdout:mds.foo.smithi089.cczuoz smithi089 running (8m) 11s ago 8m 13.9M - 17.2.7-1102-g9a202024 280b03ed6869 a1f028849336 2024-07-15T06:59:07.471 INFO:teuthology.orchestra.run.smithi089.stdout:mds.foo.smithi134.wugtzh smithi134 running (8m) 39s ago 8m 15.5M - 17.2.7-1102-g9a202024 280b03ed6869 fb5bbeafa32b 2024-07-15T06:59:07.472 INFO:teuthology.orchestra.run.smithi089.stdout:mgr.x smithi134 *:8443,9283,8765 running (2m) 39s ago 19m 422M - 18.2.2-1609-g6ea7aaa9 0add76435ed9 842a5eaedc06 2024-07-15T06:59:07.472 INFO:teuthology.orchestra.run.smithi089.stdout:mgr.y smithi089 *:8443,9283,8765 running (3m) 11s ago 21m 471M - 18.2.2-1609-g6ea7aaa9 0add76435ed9 20c56e260e99 2024-07-15T06:59:07.472 INFO:teuthology.orchestra.run.smithi089.stdout:mon.a smithi089 running (2m) 11s ago 21m 37.1M 2048M 18.2.2-1609-g6ea7aaa9 0add76435ed9 b2d2728b9d78 2024-07-15T06:59:07.472 INFO:teuthology.orchestra.run.smithi089.stdout:mon.b smithi134 running (19m) 39s ago 19m 47.6M 2048M 17.2.7-1102-g9a202024 280b03ed6869 19182ed25bde 2024-07-15T06:59:07.472 INFO:teuthology.orchestra.run.smithi089.stdout:mon.c smithi089 running (24s) 11s ago 19m 23.0M 2048M 18.2.2-1609-g6ea7aaa9 0add76435ed9 60c08b0bcea8 2024-07-15T06:59:07.472 INFO:teuthology.orchestra.run.smithi089.stdout:node-exporter.a smithi089 *:9100 running (3m) 11s ago 11m 7743k - 1.5.0 0da6a335fe13 5da5d79c182f 2024-07-15T06:59:07.472 INFO:teuthology.orchestra.run.smithi089.stdout:node-exporter.b smithi134 *:9100 running (3m) 39s ago 11m 7763k - 1.5.0 0da6a335fe13 9ace5df7006d 2024-07-15T06:59:07.472 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0 smithi089 running (18m) 11s ago 18m 49.9M 2524M 17.2.7-1102-g9a202024 280b03ed6869 3fd6dc329fe6 2024-07-15T06:59:07.472 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1 smithi089 running (17m) 11s ago 17m 49.9M 2524M 17.2.7-1102-g9a202024 280b03ed6869 b4ff988d2ed5 2024-07-15T06:59:07.472 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2 smithi089 running (16m) 11s ago 16m 56.3M 2524M 17.2.7-1102-g9a202024 280b03ed6869 428fbfd265d1 2024-07-15T06:59:07.472 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3 smithi089 running (16m) 11s ago 16m 46.7M 2524M 17.2.7-1102-g9a202024 280b03ed6869 4f43743cd007 2024-07-15T06:59:07.472 INFO:teuthology.orchestra.run.smithi089.stdout:osd.4 smithi134 running (15m) 39s ago 15m 51.0M 2524M 17.2.7-1102-g9a202024 280b03ed6869 2ae2fa51d983 2024-07-15T06:59:07.472 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5 smithi134 running (14m) 39s ago 14m 54.7M 2524M 17.2.7-1102-g9a202024 280b03ed6869 61be14c58fc3 2024-07-15T06:59:07.472 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6 smithi134 running (13m) 39s ago 13m 53.7M 2524M 17.2.7-1102-g9a202024 280b03ed6869 f92c7dd84666 2024-07-15T06:59:07.473 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7 smithi134 running (12m) 39s ago 12m 54.2M 2524M 17.2.7-1102-g9a202024 280b03ed6869 e44d11319ff6 2024-07-15T06:59:07.473 INFO:teuthology.orchestra.run.smithi089.stdout:prometheus.a smithi134 *:9095 running (2m) 39s ago 11m 52.2M - 2.43.0 a07b618ecd1d 8873e527bacf 2024-07-15T06:59:08.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:07 smithi089 bash[79810]: audit 2024-07-15T06:59:07.463456+0000 mgr.y (mgr.25244) 84 : audit [DBG] from='client.34173 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T06:59:08.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:07 smithi089 bash[82202]: audit 2024-07-15T06:59:07.463456+0000 mgr.y (mgr.25244) 84 : audit [DBG] from='client.34173 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T06:59:08.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:07 smithi134 bash[24068]: audit 2024-07-15T06:59:07.463456+0000 mgr.y (mgr.25244) 84 : audit [DBG] from='client.34173 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T06:59:08.409 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -e sha1=6ea7aaa9f93d34dc4d21862afe00140dfddbd706 -- bash -c 'ceph orch upgrade pause' 2024-07-15T06:59:09.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:08 smithi134 bash[24068]: cluster 2024-07-15T06:59:08.746625+0000 mgr.y (mgr.25244) 85 : cluster [DBG] pgmap v65: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:59:09.054 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:59:08 smithi134 bash[60729]: ts=2024-07-15T06:59:08.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1102-g9a202024 (9a2020246f85463d4fdb115c94aa40b463caa1d2) quincy (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1102-g9a202024 (9a2020246f85463d4fdb115c94aa40b463caa1d2) quincy (stable)\", cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:59:09.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:08 smithi089 bash[79810]: cluster 2024-07-15T06:59:08.746625+0000 mgr.y (mgr.25244) 85 : cluster [DBG] pgmap v65: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:59:09.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:08 smithi089 bash[82202]: cluster 2024-07-15T06:59:08.746625+0000 mgr.y (mgr.25244) 85 : cluster [DBG] pgmap v65: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:59:10.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:09 smithi089 bash[79810]: audit 2024-07-15T06:59:08.853445+0000 mon.a (mon.0) 101 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:59:10.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:09 smithi089 bash[79810]: audit 2024-07-15T06:59:08.860368+0000 mon.a (mon.0) 102 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:59:10.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:09 smithi089 bash[79810]: audit 2024-07-15T06:59:08.862018+0000 mon.a (mon.0) 103 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:59:10.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:09 smithi089 bash[79810]: audit 2024-07-15T06:59:08.863237+0000 mon.a (mon.0) 104 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:59:10.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:09 smithi089 bash[79810]: audit 2024-07-15T06:59:08.872024+0000 mon.a (mon.0) 105 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:59:10.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:09 smithi089 bash[79810]: audit 2024-07-15T06:59:09.020455+0000 mon.a (mon.0) 106 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:59:10.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:09 smithi089 bash[79810]: audit 2024-07-15T06:59:09.024738+0000 mon.a (mon.0) 107 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-15T06:59:10.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:09 smithi089 bash[79810]: audit 2024-07-15T06:59:09.027102+0000 mon.a (mon.0) 108 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-07-15T06:59:10.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:09 smithi089 bash[79810]: audit 2024-07-15T06:59:09.028686+0000 mon.a (mon.0) 109 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-07-15T06:59:10.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:09 smithi089 bash[79810]: cephadm 2024-07-15T06:59:09.029665+0000 mgr.y (mgr.25244) 86 : cephadm [INF] Upgrade: It appears safe to stop mon.b 2024-07-15T06:59:10.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:09 smithi089 bash[82202]: audit 2024-07-15T06:59:08.853445+0000 mon.a (mon.0) 101 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:59:10.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:09 smithi089 bash[82202]: audit 2024-07-15T06:59:08.860368+0000 mon.a (mon.0) 102 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:59:10.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:09 smithi089 bash[82202]: audit 2024-07-15T06:59:08.862018+0000 mon.a (mon.0) 103 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:59:10.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:09 smithi089 bash[82202]: audit 2024-07-15T06:59:08.863237+0000 mon.a (mon.0) 104 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:59:10.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:09 smithi089 bash[82202]: audit 2024-07-15T06:59:08.872024+0000 mon.a (mon.0) 105 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:59:10.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:09 smithi089 bash[82202]: audit 2024-07-15T06:59:09.020455+0000 mon.a (mon.0) 106 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:59:10.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:09 smithi089 bash[82202]: audit 2024-07-15T06:59:09.024738+0000 mon.a (mon.0) 107 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-15T06:59:10.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:09 smithi089 bash[82202]: audit 2024-07-15T06:59:09.027102+0000 mon.a (mon.0) 108 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-07-15T06:59:10.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:09 smithi089 bash[82202]: audit 2024-07-15T06:59:09.028686+0000 mon.a (mon.0) 109 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-07-15T06:59:10.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:09 smithi089 bash[82202]: cephadm 2024-07-15T06:59:09.029665+0000 mgr.y (mgr.25244) 86 : cephadm [INF] Upgrade: It appears safe to stop mon.b 2024-07-15T06:59:10.304 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:59:10 smithi134 bash[60729]: ts=2024-07-15T06:59:10.087Z caller=alerting.go:366 level=warn component="rule manager" alert="unsupported value type" msg="Expanding alert template failed" err="error executing template __alert_CephMonDown: template: __alert_CephMonDown:1:427: executing \"__alert_CephMonDown\" at : error calling query: found duplicate series for the match group {ceph_daemon=\"mon.a\"} on the right hand-side of the operation: [{ceph_daemon=\"mon.a\", ceph_version=\"ceph version 18.2.2-1609-g6ea7aaa9 (6ea7aaa9f93d34dc4d21862afe00140dfddbd706) reef (stable)\", cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", hostname=\"smithi089\", instance=\"ceph_cluster\", job=\"ceph\", public_addr=\"172.21.15.89\", rank=\"0\"}, {ceph_daemon=\"mon.a\", ceph_version=\"ceph version 17.2.7-1102-g9a202024 (9a2020246f85463d4fdb115c94aa40b463caa1d2) quincy (stable)\", hostname=\"smithi089\", instance=\"ceph_cluster\", job=\"ceph\", public_addr=\"172.21.15.89\", rank=\"0\"}];many-to-many matching not allowed: matching labels must be unique on one side" data="unsupported value type" 2024-07-15T06:59:10.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:09 smithi134 bash[24068]: audit 2024-07-15T06:59:08.853445+0000 mon.a (mon.0) 101 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:59:10.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:09 smithi134 bash[24068]: audit 2024-07-15T06:59:08.860368+0000 mon.a (mon.0) 102 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:59:10.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:09 smithi134 bash[24068]: audit 2024-07-15T06:59:08.862018+0000 mon.a (mon.0) 103 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:59:10.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:09 smithi134 bash[24068]: audit 2024-07-15T06:59:08.863237+0000 mon.a (mon.0) 104 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:59:10.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:09 smithi134 bash[24068]: audit 2024-07-15T06:59:08.872024+0000 mon.a (mon.0) 105 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:59:10.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:09 smithi134 bash[24068]: audit 2024-07-15T06:59:09.020455+0000 mon.a (mon.0) 106 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T06:59:10.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:09 smithi134 bash[24068]: audit 2024-07-15T06:59:09.024738+0000 mon.a (mon.0) 107 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-15T06:59:10.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:09 smithi134 bash[24068]: audit 2024-07-15T06:59:09.027102+0000 mon.a (mon.0) 108 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-07-15T06:59:10.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:09 smithi134 bash[24068]: audit 2024-07-15T06:59:09.028686+0000 mon.a (mon.0) 109 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-07-15T06:59:10.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:09 smithi134 bash[24068]: cephadm 2024-07-15T06:59:09.029665+0000 mgr.y (mgr.25244) 86 : cephadm [INF] Upgrade: It appears safe to stop mon.b 2024-07-15T06:59:11.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:10 smithi089 bash[79810]: cluster 2024-07-15T06:59:10.747333+0000 mgr.y (mgr.25244) 87 : cluster [DBG] pgmap v66: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:59:11.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:10 smithi089 bash[82202]: cluster 2024-07-15T06:59:10.747333+0000 mgr.y (mgr.25244) 87 : cluster [DBG] pgmap v66: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:59:11.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:10 smithi134 bash[24068]: cluster 2024-07-15T06:59:10.747333+0000 mgr.y (mgr.25244) 87 : cluster [DBG] pgmap v66: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:59:12.733 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:59:12 smithi134 bash[60729]: ts=2024-07-15T06:59:12.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:59:12.733 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:12 smithi134 bash[24068]: cephadm 2024-07-15T06:59:11.434660+0000 mgr.y (mgr.25244) 88 : cephadm [INF] Upgrade: Updating mon.b 2024-07-15T06:59:12.733 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:12 smithi134 bash[24068]: audit 2024-07-15T06:59:11.447178+0000 mon.a (mon.0) 110 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:59:12.733 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:12 smithi134 bash[24068]: audit 2024-07-15T06:59:11.449196+0000 mon.a (mon.0) 111 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-07-15T06:59:12.733 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:12 smithi134 bash[24068]: audit 2024-07-15T06:59:11.450909+0000 mon.a (mon.0) 112 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-07-15T06:59:12.733 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:12 smithi134 bash[24068]: audit 2024-07-15T06:59:11.452662+0000 mon.a (mon.0) 113 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:59:12.734 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:12 smithi134 bash[24068]: cephadm 2024-07-15T06:59:11.454606+0000 mgr.y (mgr.25244) 89 : cephadm [INF] Deploying daemon mon.b on smithi134 2024-07-15T06:59:12.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:12 smithi089 bash[79810]: cephadm 2024-07-15T06:59:11.434660+0000 mgr.y (mgr.25244) 88 : cephadm [INF] Upgrade: Updating mon.b 2024-07-15T06:59:12.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:12 smithi089 bash[79810]: audit 2024-07-15T06:59:11.447178+0000 mon.a (mon.0) 110 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:59:12.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:12 smithi089 bash[79810]: audit 2024-07-15T06:59:11.449196+0000 mon.a (mon.0) 111 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-07-15T06:59:12.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:12 smithi089 bash[79810]: audit 2024-07-15T06:59:11.450909+0000 mon.a (mon.0) 112 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-07-15T06:59:12.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:12 smithi089 bash[79810]: audit 2024-07-15T06:59:11.452662+0000 mon.a (mon.0) 113 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:59:12.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:12 smithi089 bash[79810]: cephadm 2024-07-15T06:59:11.454606+0000 mgr.y (mgr.25244) 89 : cephadm [INF] Deploying daemon mon.b on smithi134 2024-07-15T06:59:12.870 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:12 smithi089 bash[82202]: cephadm 2024-07-15T06:59:11.434660+0000 mgr.y (mgr.25244) 88 : cephadm [INF] Upgrade: Updating mon.b 2024-07-15T06:59:12.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:12 smithi089 bash[82202]: audit 2024-07-15T06:59:11.447178+0000 mon.a (mon.0) 110 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:59:12.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:12 smithi089 bash[82202]: audit 2024-07-15T06:59:11.449196+0000 mon.a (mon.0) 111 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-07-15T06:59:12.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:12 smithi089 bash[82202]: audit 2024-07-15T06:59:11.450909+0000 mon.a (mon.0) 112 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-07-15T06:59:12.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:12 smithi089 bash[82202]: audit 2024-07-15T06:59:11.452662+0000 mon.a (mon.0) 113 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:59:12.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:12 smithi089 bash[82202]: cephadm 2024-07-15T06:59:11.454606+0000 mgr.y (mgr.25244) 89 : cephadm [INF] Deploying daemon mon.b on smithi134 2024-07-15T06:59:13.194 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T06:59:13.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:13 smithi089 bash[79810]: cluster 2024-07-15T06:59:12.748353+0000 mgr.y (mgr.25244) 90 : cluster [DBG] pgmap v67: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:59:13.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:13 smithi089 bash[82202]: cluster 2024-07-15T06:59:12.748353+0000 mgr.y (mgr.25244) 90 : cluster [DBG] pgmap v67: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:59:13.802 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:13 smithi134 bash[24068]: cluster 2024-07-15T06:59:12.748353+0000 mgr.y (mgr.25244) 90 : cluster [DBG] pgmap v67: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:59:13.802 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:59:13 smithi134 bash[60729]: ts=2024-07-15T06:59:13.508Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi134\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi134\", job=\"node\", machine=\"x86_64\", nodename=\"smithi134\", release=\"5.15.0-113-generic\", sysname=\"Linux\", version=\"#123-Ubuntu SMP Mon Jun 10 08:16:17 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", domainname=\"(none)\", instance=\"smithi134\", job=\"node\", machine=\"x86_64\", nodename=\"smithi134\", release=\"5.15.0-113-generic\", sysname=\"Linux\", version=\"#123-Ubuntu SMP Mon Jun 10 08:16:17 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:59:14.054 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:59:13 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:59:14.054 INFO:journalctl@ceph.osd.4.smithi134.stdout:Jul 15 06:59:13 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:59:14.055 INFO:journalctl@ceph.osd.5.smithi134.stdout:Jul 15 06:59:13 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:59:14.055 INFO:journalctl@ceph.osd.6.smithi134.stdout:Jul 15 06:59:13 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:59:14.055 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 06:59:13 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:59:14.055 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:59:13 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:59:14.056 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:59:13 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:59:14.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:13 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:59:14.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:13 smithi134 systemd[1]: Stopping Ceph mon.b for 75ba02bc-4274-11ef-bcad-c7b262605968... 2024-07-15T06:59:14.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:14 smithi134 bash[24068]: debug 2024-07-15T06:59:14.029+0000 7fc5f3abe640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2024-07-15T06:59:14.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:14 smithi134 bash[24068]: debug 2024-07-15T06:59:14.029+0000 7fc5f3abe640 -1 mon.b@1(peon) e3 *** Got Signal Terminated *** 2024-07-15T06:59:14.057 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:59:13 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:59:14.477 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:59:14 smithi089 bash[72894]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:59:14] "GET /metrics HTTP/1.1" 200 33854 "" "Prometheus/2.43.0" 2024-07-15T06:59:14.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:14 smithi134 bash[67506]: ceph-75ba02bc-4274-11ef-bcad-c7b262605968-mon-b 2024-07-15T06:59:14.804 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:59:14 smithi134 bash[60729]: ts=2024-07-15T06:59:14.454Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:59:14.804 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:59:14 smithi134 bash[60729]: ts=2024-07-15T06:59:14.455Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:59:15.162 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:14 smithi134 bash[67568]: Error response from daemon: No such container: ceph-75ba02bc-4274-11ef-bcad-c7b262605968-mon-b 2024-07-15T06:59:15.162 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:14 smithi134 systemd[1]: ceph-75ba02bc-4274-11ef-bcad-c7b262605968@mon.b.service: Deactivated successfully. 2024-07-15T06:59:15.162 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:14 smithi134 systemd[1]: Stopped Ceph mon.b for 75ba02bc-4274-11ef-bcad-c7b262605968. 2024-07-15T06:59:15.162 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:15 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:59:15.163 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 06:59:15 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:59:15.163 INFO:journalctl@ceph.osd.4.smithi134.stdout:Jul 15 06:59:15 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:59:15.163 INFO:journalctl@ceph.osd.5.smithi134.stdout:Jul 15 06:59:15 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:59:15.164 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 06:59:15 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:59:15.164 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 06:59:15 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:59:15.164 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 06:59:15 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:59:15.164 INFO:journalctl@ceph.osd.6.smithi134.stdout:Jul 15 06:59:15 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:59:15.165 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:59:15 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T06:59:15.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:15 smithi134 systemd[1]: Started Ceph mon.b for 75ba02bc-4274-11ef-bcad-c7b262605968. 2024-07-15T06:59:16.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.357+0000 7f2235b3db00 0 set uid:gid to 167:167 (ceph:ceph) 2024-07-15T06:59:16.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.357+0000 7f2235b3db00 0 ceph version 18.2.2-1609-g6ea7aaa9 (6ea7aaa9f93d34dc4d21862afe00140dfddbd706) reef (stable), process ceph-mon, pid 7 2024-07-15T06:59:16.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.357+0000 7f2235b3db00 0 pidfile_write: ignore empty --pid-file 2024-07-15T06:59:16.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 0 load: jerasure load: lrc 2024-07-15T06:59:16.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: RocksDB version: 7.9.2 2024-07-15T06:59:16.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Git sha 0 2024-07-15T06:59:16.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Compile date 2024-07-05 15:13:19 2024-07-15T06:59:16.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: DB SUMMARY 2024-07-15T06:59:16.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: DB Session ID: WRX2SAI2OFG7IKNTUZ7C 2024-07-15T06:59:16.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: CURRENT file: CURRENT 2024-07-15T06:59:16.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: IDENTITY file: IDENTITY 2024-07-15T06:59:16.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: MANIFEST file: MANIFEST-000009 size: 1659 Bytes 2024-07-15T06:59:16.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 1, files: 000036.sst 2024-07-15T06:59:16.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000034.log size: 4694960 ; 2024-07-15T06:59:16.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.error_if_exists: 0 2024-07-15T06:59:16.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.create_if_missing: 0 2024-07-15T06:59:16.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.paranoid_checks: 1 2024-07-15T06:59:16.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.flush_verify_memtable_count: 1 2024-07-15T06:59:16.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-07-15T06:59:16.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-07-15T06:59:16.557 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.env: 0x55c8c32acba0 2024-07-15T06:59:16.557 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.fs: PosixFileSystem 2024-07-15T06:59:16.557 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.info_log: 0x55c8c46febe0 2024-07-15T06:59:16.557 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.max_file_opening_threads: 16 2024-07-15T06:59:16.557 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.statistics: (nil) 2024-07-15T06:59:16.557 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.use_fsync: 0 2024-07-15T06:59:16.557 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.max_log_file_size: 0 2024-07-15T06:59:16.558 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.max_manifest_file_size: 1073741824 2024-07-15T06:59:16.558 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.log_file_time_to_roll: 0 2024-07-15T06:59:16.558 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.keep_log_file_num: 1000 2024-07-15T06:59:16.558 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.recycle_log_file_num: 0 2024-07-15T06:59:16.558 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.allow_fallocate: 1 2024-07-15T06:59:16.558 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.allow_mmap_reads: 0 2024-07-15T06:59:16.558 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.allow_mmap_writes: 0 2024-07-15T06:59:16.558 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.use_direct_reads: 0 2024-07-15T06:59:16.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-07-15T06:59:16.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.create_missing_column_families: 0 2024-07-15T06:59:16.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.db_log_dir: 2024-07-15T06:59:16.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.wal_dir: 2024-07-15T06:59:16.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.table_cache_numshardbits: 6 2024-07-15T06:59:16.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.WAL_ttl_seconds: 0 2024-07-15T06:59:16.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.WAL_size_limit_MB: 0 2024-07-15T06:59:16.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-07-15T06:59:16.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.manifest_preallocation_size: 4194304 2024-07-15T06:59:16.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.is_fd_close_on_exec: 1 2024-07-15T06:59:16.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.advise_random_on_open: 1 2024-07-15T06:59:16.560 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.db_write_buffer_size: 0 2024-07-15T06:59:16.560 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.write_buffer_manager: 0x55c8c470cb40 2024-07-15T06:59:16.560 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.access_hint_on_compaction_start: 1 2024-07-15T06:59:16.560 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2024-07-15T06:59:16.560 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.use_adaptive_mutex: 0 2024-07-15T06:59:16.560 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.rate_limiter: (nil) 2024-07-15T06:59:16.560 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-07-15T06:59:16.560 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.wal_recovery_mode: 2 2024-07-15T06:59:16.560 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.enable_thread_tracking: 0 2024-07-15T06:59:16.560 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.enable_pipelined_write: 0 2024-07-15T06:59:16.560 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.unordered_write: 0 2024-07-15T06:59:16.560 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2024-07-15T06:59:16.560 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-07-15T06:59:16.561 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.write_thread_max_yield_usec: 100 2024-07-15T06:59:16.561 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2024-07-15T06:59:16.561 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.row_cache: None 2024-07-15T06:59:16.561 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.wal_filter: None 2024-07-15T06:59:16.561 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.avoid_flush_during_recovery: 0 2024-07-15T06:59:16.561 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.allow_ingest_behind: 0 2024-07-15T06:59:16.561 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.two_write_queues: 0 2024-07-15T06:59:16.561 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.manual_wal_flush: 0 2024-07-15T06:59:16.561 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.wal_compression: 0 2024-07-15T06:59:16.561 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.atomic_flush: 0 2024-07-15T06:59:16.561 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-07-15T06:59:16.561 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.persist_stats_to_disk: 0 2024-07-15T06:59:16.562 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.write_dbid_to_manifest: 0 2024-07-15T06:59:16.562 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.log_readahead_size: 0 2024-07-15T06:59:16.562 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.file_checksum_gen_factory: Unknown 2024-07-15T06:59:16.562 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.best_efforts_recovery: 0 2024-07-15T06:59:16.562 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-07-15T06:59:16.562 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-07-15T06:59:16.562 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.allow_data_in_errors: 0 2024-07-15T06:59:16.562 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.db_host_id: __hostname__ 2024-07-15T06:59:16.562 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.enforce_single_del_contracts: true 2024-07-15T06:59:16.562 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.max_background_jobs: 2 2024-07-15T06:59:16.562 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.max_background_compactions: -1 2024-07-15T06:59:16.562 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.max_subcompactions: 1 2024-07-15T06:59:16.563 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2024-07-15T06:59:16.563 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-07-15T06:59:16.563 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.delayed_write_rate : 16777216 2024-07-15T06:59:16.563 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.max_total_wal_size: 0 2024-07-15T06:59:16.563 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-07-15T06:59:16.563 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.stats_dump_period_sec: 600 2024-07-15T06:59:16.563 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.stats_persist_period_sec: 600 2024-07-15T06:59:16.563 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.stats_history_buffer_size: 1048576 2024-07-15T06:59:16.563 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.max_open_files: -1 2024-07-15T06:59:16.563 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.bytes_per_sync: 0 2024-07-15T06:59:16.563 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.wal_bytes_per_sync: 0 2024-07-15T06:59:16.563 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.strict_bytes_per_sync: 0 2024-07-15T06:59:16.563 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.compaction_readahead_size: 0 2024-07-15T06:59:16.564 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.max_background_flushes: -1 2024-07-15T06:59:16.564 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Compression algorithms supported: 2024-07-15T06:59:16.564 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: kZSTD supported: 0 2024-07-15T06:59:16.564 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: kXpressCompression supported: 0 2024-07-15T06:59:16.564 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: kBZip2Compression supported: 0 2024-07-15T06:59:16.564 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: kZSTDNotFinalCompression supported: 0 2024-07-15T06:59:16.564 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: kLZ4Compression supported: 1 2024-07-15T06:59:16.564 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: kZlibCompression supported: 1 2024-07-15T06:59:16.564 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: kLZ4HCCompression supported: 1 2024-07-15T06:59:16.564 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: kSnappyCompression supported: 1 2024-07-15T06:59:16.564 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Fast CRC32 supported: Supported on x86 2024-07-15T06:59:16.564 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: DMutex implementation: pthread_mutex_t 2024-07-15T06:59:16.564 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 2024-07-15T06:59:16.566 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-07-15T06:59:16.566 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-07-15T06:59:16.566 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.merge_operator: 2024-07-15T06:59:16.566 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.compaction_filter: None 2024-07-15T06:59:16.566 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.compaction_filter_factory: None 2024-07-15T06:59:16.566 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.sst_partitioner_factory: None 2024-07-15T06:59:16.566 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.memtable_factory: SkipListFactory 2024-07-15T06:59:16.566 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.table_factory: BlockBasedTable 2024-07-15T06:59:16.566 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55c8c46fe820) 2024-07-15T06:59:16.567 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: cache_index_and_filter_blocks: 1 2024-07-15T06:59:16.567 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: cache_index_and_filter_blocks_with_high_priority: 0 2024-07-15T06:59:16.567 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: pin_l0_filter_and_index_blocks_in_cache: 0 2024-07-15T06:59:16.567 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: pin_top_level_index_and_filter: 1 2024-07-15T06:59:16.567 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: index_type: 0 2024-07-15T06:59:16.567 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: data_block_index_type: 0 2024-07-15T06:59:16.567 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: index_shortening: 1 2024-07-15T06:59:16.567 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: data_block_hash_table_util_ratio: 0.750000 2024-07-15T06:59:16.567 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: checksum: 4 2024-07-15T06:59:16.567 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: no_block_cache: 0 2024-07-15T06:59:16.567 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: block_cache: 0x55c8c46f3350 2024-07-15T06:59:16.567 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: block_cache_name: BinnedLRUCache 2024-07-15T06:59:16.567 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: block_cache_options: 2024-07-15T06:59:16.568 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: capacity : 536870912 2024-07-15T06:59:16.568 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: num_shard_bits : 4 2024-07-15T06:59:16.568 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: strict_capacity_limit : 0 2024-07-15T06:59:16.568 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: high_pri_pool_ratio: 0.000 2024-07-15T06:59:16.568 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: block_cache_compressed: (nil) 2024-07-15T06:59:16.568 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: persistent_cache: (nil) 2024-07-15T06:59:16.568 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: block_size: 4096 2024-07-15T06:59:16.568 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: block_size_deviation: 10 2024-07-15T06:59:16.568 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: block_restart_interval: 16 2024-07-15T06:59:16.568 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: index_block_restart_interval: 1 2024-07-15T06:59:16.568 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: metadata_block_size: 4096 2024-07-15T06:59:16.568 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: partition_filters: 0 2024-07-15T06:59:16.568 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: use_delta_encoding: 1 2024-07-15T06:59:16.568 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: filter_policy: bloomfilter 2024-07-15T06:59:16.569 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: whole_key_filtering: 1 2024-07-15T06:59:16.569 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: verify_compression: 0 2024-07-15T06:59:16.569 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: read_amp_bytes_per_bit: 0 2024-07-15T06:59:16.569 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: format_version: 5 2024-07-15T06:59:16.569 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: enable_index_compression: 1 2024-07-15T06:59:16.569 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: block_align: 0 2024-07-15T06:59:16.569 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: max_auto_readahead_size: 262144 2024-07-15T06:59:16.569 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: prepopulate_block_cache: 0 2024-07-15T06:59:16.569 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: initial_auto_readahead_size: 8192 2024-07-15T06:59:16.569 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: num_file_reads_for_auto_readahead: 2 2024-07-15T06:59:16.569 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.write_buffer_size: 33554432 2024-07-15T06:59:16.569 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.max_write_buffer_number: 2 2024-07-15T06:59:16.569 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.compression: NoCompression 2024-07-15T06:59:16.569 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.bottommost_compression: Disabled 2024-07-15T06:59:16.570 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.prefix_extractor: nullptr 2024-07-15T06:59:16.570 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-07-15T06:59:16.570 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.num_levels: 7 2024-07-15T06:59:16.570 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-07-15T06:59:16.570 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-07-15T06:59:16.570 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-07-15T06:59:16.570 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-07-15T06:59:16.570 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2024-07-15T06:59:16.570 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-07-15T06:59:16.570 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-07-15T06:59:16.570 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-07-15T06:59:16.570 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-07-15T06:59:16.570 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2024-07-15T06:59:16.570 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-07-15T06:59:16.571 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-07-15T06:59:16.571 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.compression_opts.window_bits: -14 2024-07-15T06:59:16.571 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.compression_opts.level: 32767 2024-07-15T06:59:16.571 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.compression_opts.strategy: 0 2024-07-15T06:59:16.571 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-07-15T06:59:16.571 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-07-15T06:59:16.571 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-07-15T06:59:16.571 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.compression_opts.parallel_threads: 1 2024-07-15T06:59:16.571 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.compression_opts.enabled: false 2024-07-15T06:59:16.571 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-07-15T06:59:16.571 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-07-15T06:59:16.571 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-07-15T06:59:16.571 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.level0_stop_writes_trigger: 36 2024-07-15T06:59:16.571 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.target_file_size_base: 67108864 2024-07-15T06:59:16.572 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.target_file_size_multiplier: 1 2024-07-15T06:59:16.572 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2024-07-15T06:59:16.572 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-07-15T06:59:16.572 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-07-15T06:59:16.572 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-07-15T06:59:16.572 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-07-15T06:59:16.572 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-07-15T06:59:16.572 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-07-15T06:59:16.572 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-07-15T06:59:16.572 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-07-15T06:59:16.572 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-07-15T06:59:16.572 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-07-15T06:59:16.572 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.max_compaction_bytes: 1677721600 2024-07-15T06:59:16.573 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-07-15T06:59:16.573 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.arena_block_size: 1048576 2024-07-15T06:59:16.573 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-07-15T06:59:16.573 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-07-15T06:59:16.573 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.disable_auto_compactions: 0 2024-07-15T06:59:16.573 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-07-15T06:59:16.573 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-07-15T06:59:16.573 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-07-15T06:59:16.573 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-07-15T06:59:16.573 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-07-15T06:59:16.573 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-07-15T06:59:16.573 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-07-15T06:59:16.574 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-07-15T06:59:16.574 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-07-15T06:59:16.574 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-07-15T06:59:16.574 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.table_properties_collectors: 2024-07-15T06:59:16.574 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.inplace_update_support: 0 2024-07-15T06:59:16.574 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.inplace_update_num_locks: 10000 2024-07-15T06:59:16.574 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-07-15T06:59:16.574 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.memtable_whole_key_filtering: 0 2024-07-15T06:59:16.574 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.memtable_huge_page_size: 0 2024-07-15T06:59:16.574 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.bloom_locality: 0 2024-07-15T06:59:16.574 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.max_successive_merges: 0 2024-07-15T06:59:16.574 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.optimize_filters_for_hits: 0 2024-07-15T06:59:16.574 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.paranoid_file_checks: 0 2024-07-15T06:59:16.575 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.force_consistency_checks: 1 2024-07-15T06:59:16.575 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.report_bg_io_stats: 0 2024-07-15T06:59:16.575 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.ttl: 2592000 2024-07-15T06:59:16.575 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.periodic_compaction_seconds: 0 2024-07-15T06:59:16.575 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.preclude_last_level_data_seconds: 0 2024-07-15T06:59:16.575 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.preserve_internal_time_seconds: 0 2024-07-15T06:59:16.575 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.enable_blob_files: false 2024-07-15T06:59:16.575 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.min_blob_size: 0 2024-07-15T06:59:16.575 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.blob_file_size: 268435456 2024-07-15T06:59:16.575 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.blob_compression_type: NoCompression 2024-07-15T06:59:16.575 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.enable_blob_garbage_collection: false 2024-07-15T06:59:16.575 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-07-15T06:59:16.575 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-07-15T06:59:16.576 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.blob_compaction_readahead_size: 0 2024-07-15T06:59:16.576 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.blob_file_starting_level: 0 2024-07-15T06:59:16.576 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-07-15T06:59:16.576 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2232916640 3 rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 36.sst 2024-07-15T06:59:16.576 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-07-15T06:59:16.576 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 38, last_sequence is 18040, log_number is 34,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-07-15T06:59:16.576 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.361+0000 7f2235b3db00 4 rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 34 2024-07-15T06:59:16.576 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.365+0000 7f2235b3db00 4 rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 2c2e6b12-bb96-406c-92a7-bbe7752887ff 2024-07-15T06:59:16.576 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.365+0000 7f2235b3db00 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1721026756369448, "job": 1, "event": "recovery_started", "wal_files": [34]} 2024-07-15T06:59:16.576 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.365+0000 7f2235b3db00 4 rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #34 mode 2 2024-07-15T06:59:16.576 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.377+0000 7f2235b3db00 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1721026756383367, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 39, "file_size": 2931172, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 18045, "largest_seqno": 19961, "table_properties": {"data_size": 2923101, "index_size": 5195, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1861, "raw_key_size": 16610, "raw_average_key_size": 22, "raw_value_size": 2907182, "raw_average_value_size": 3960, "num_data_blocks": 235, "num_entries": 734, "num_filter_entries": 734, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1721026756, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "2c2e6b12-bb96-406c-92a7-bbe7752887ff", "db_session_id": "WRX2SAI2OFG7IKNTUZ7C", "orig_file_number": 39, "seqno_to_time_mapping": "N/A"}} 2024-07-15T06:59:16.576 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.377+0000 7f2235b3db00 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1721026756383509, "job": 1, "event": "recovery_finished"} 2024-07-15T06:59:16.576 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.377+0000 7f2235b3db00 4 rocksdb: [db/version_set.cc:5047] Creating manifest 41 2024-07-15T06:59:16.576 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.377+0000 7f2235b3db00 4 rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-07-15T06:59:16.576 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.377+0000 7f2235b3db00 4 rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000034.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-07-15T06:59:16.577 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.377+0000 7f2235b3db00 4 rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55c8c471ee00 2024-07-15T06:59:16.577 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.377+0000 7f2235b3db00 4 rocksdb: DB pointer 0x55c8c4816000 2024-07-15T06:59:16.577 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.381+0000 7f222b908640 4 rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-07-15T06:59:16.577 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.381+0000 7f222b908640 4 rocksdb: [db/db_impl/db_impl.cc:1111] 2024-07-15T06:59:16.577 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: ** DB Stats ** 2024-07-15T06:59:16.577 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: Uptime(secs): 0.0 total, 0.0 interval 2024-07-15T06:59:16.577 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-07-15T06:59:16.577 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-07-15T06:59:16.577 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-07-15T06:59:16.577 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-07-15T06:59:16.577 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-07-15T06:59:16.577 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-07-15T06:59:16.577 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: ** Compaction Stats [default] ** 2024-07-15T06:59:16.577 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-07-15T06:59:16.578 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-07-15T06:59:16.578 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: L0 1/0 2.80 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 525.9 0.01 0.00 1 0.005 0 0 0.0 0.0 2024-07-15T06:59:16.578 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: L6 1/0 7.17 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-07-15T06:59:16.578 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: Sum 2/0 9.96 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 525.9 0.01 0.00 1 0.005 0 0 0.0 0.0 2024-07-15T06:59:16.578 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 525.9 0.01 0.00 1 0.005 0 0 0.0 0.0 2024-07-15T06:59:16.578 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: ** Compaction Stats [default] ** 2024-07-15T06:59:16.578 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-07-15T06:59:16.578 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-07-15T06:59:16.578 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 525.9 0.01 0.00 1 0.005 0 0 0.0 0.0 2024-07-15T06:59:16.578 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-07-15T06:59:16.578 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: Uptime(secs): 0.0 total, 0.0 interval 2024-07-15T06:59:16.578 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: Flush(GB): cumulative 0.003, interval 0.003 2024-07-15T06:59:16.578 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: AddFile(GB): cumulative 0.000, interval 0.000 2024-07-15T06:59:16.578 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: AddFile(Total Files): cumulative 0, interval 0 2024-07-15T06:59:16.579 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: AddFile(L0 Files): cumulative 0, interval 0 2024-07-15T06:59:16.579 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: AddFile(Keys): cumulative 0, interval 0 2024-07-15T06:59:16.579 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: Cumulative compaction: 0.00 GB write, 163.42 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-07-15T06:59:16.579 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: Interval compaction: 0.00 GB write, 163.42 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-07-15T06:59:16.579 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-07-15T06:59:16.579 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: Block cache BinnedLRUCache@0x55c8c46f3350#7 capacity: 512.00 MB usage: 8.11 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.5e-05 secs_since: 0 2024-07-15T06:59:16.579 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: Block cache entry stats(count,size,portion): FilterBlock(1,2.03 KB,0.00038743%) IndexBlock(1,6.08 KB,0.00115931%) Misc(1,0.00 KB,0%) 2024-07-15T06:59:16.579 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: ** File Read Latency Histogram By Level [default] ** 2024-07-15T06:59:16.579 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.381+0000 7f2235b3db00 0 starting mon.b rank 1 at public addrs [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] at bind addrs [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid 75ba02bc-4274-11ef-bcad-c7b262605968 2024-07-15T06:59:16.579 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.381+0000 7f2235b3db00 1 mon.b@-1(???) e3 preinit fsid 75ba02bc-4274-11ef-bcad-c7b262605968 2024-07-15T06:59:16.579 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.381+0000 7f2235b3db00 0 mon.b@-1(???).mds e10 new map 2024-07-15T06:59:16.579 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.381+0000 7f2235b3db00 0 mon.b@-1(???).mds e10 print_map 2024-07-15T06:59:16.580 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: e10 2024-07-15T06:59:16.580 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: enable_multiple, ever_enabled_multiple: 1,1 2024-07-15T06:59:16.580 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-07-15T06:59:16.580 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: legacy client fscid: 1 2024-07-15T06:59:16.580 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: 2024-07-15T06:59:16.580 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: Filesystem 'foo' (1) 2024-07-15T06:59:16.580 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: fs_name foo 2024-07-15T06:59:16.580 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: epoch 10 2024-07-15T06:59:16.580 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: flags 12 joinable allow_snaps allow_multimds_snaps 2024-07-15T06:59:16.580 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: created 2024-07-15T06:50:29.283802+0000 2024-07-15T06:59:16.580 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: modified 2024-07-15T06:50:39.720043+0000 2024-07-15T06:59:16.580 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: tableserver 0 2024-07-15T06:59:16.580 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: root 0 2024-07-15T06:59:16.580 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: session_timeout 60 2024-07-15T06:59:16.581 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: session_autoclose 300 2024-07-15T06:59:16.581 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: max_file_size 1099511627776 2024-07-15T06:59:16.581 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: max_xattr_size 65536 2024-07-15T06:59:16.581 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: required_client_features {} 2024-07-15T06:59:16.581 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: last_failure 0 2024-07-15T06:59:16.581 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: last_failure_osd_epoch 68 2024-07-15T06:59:16.581 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-07-15T06:59:16.581 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: max_mds 1 2024-07-15T06:59:16.581 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: in 0 2024-07-15T06:59:16.581 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: up {0=24415} 2024-07-15T06:59:16.581 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: failed 2024-07-15T06:59:16.581 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: damaged 2024-07-15T06:59:16.582 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: stopped 2024-07-15T06:59:16.582 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: data_pools [4] 2024-07-15T06:59:16.582 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: metadata_pool 3 2024-07-15T06:59:16.582 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: inline_data disabled 2024-07-15T06:59:16.582 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: balancer 2024-07-15T06:59:16.582 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: bal_rank_mask -1 2024-07-15T06:59:16.582 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: standby_count_wanted 1 2024-07-15T06:59:16.582 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: [mds.foo.smithi134.wugtzh{0:24415} state up:active seq 4 join_fscid=1 addr [v2:172.21.15.134:6832/968641534,v1:172.21.15.134:6833/968641534] compat {c=[1],r=[1],i=[7ff]}] 2024-07-15T06:59:16.582 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: 2024-07-15T06:59:16.582 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: 2024-07-15T06:59:16.582 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: Standby daemons: 2024-07-15T06:59:16.582 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: 2024-07-15T06:59:16.583 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: [mds.foo.smithi089.cczuoz{-1:24488} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.89:6834/1598068998,v1:172.21.15.89:6835/1598068998] compat {c=[1],r=[1],i=[7ff]}] 2024-07-15T06:59:16.583 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.381+0000 7f2235b3db00 0 mon.b@-1(???).osd e79 crush map has features 3314933000854323200, adjusting msgr requires 2024-07-15T06:59:16.583 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.381+0000 7f2235b3db00 0 mon.b@-1(???).osd e79 crush map has features 432629239337189376, adjusting msgr requires 2024-07-15T06:59:16.583 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.381+0000 7f2235b3db00 0 mon.b@-1(???).osd e79 crush map has features 432629239337189376, adjusting msgr requires 2024-07-15T06:59:16.583 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.381+0000 7f2235b3db00 0 mon.b@-1(???).osd e79 crush map has features 432629239337189376, adjusting msgr requires 2024-07-15T06:59:16.583 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:16 smithi134 bash[67624]: debug 2024-07-15T06:59:16.381+0000 7f2235b3db00 1 mon.b@-1(???).paxosservice(auth 1..24) refresh upgraded, format 0 -> 3 2024-07-15T06:59:18.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:17 smithi089 bash[79810]: debug 2024-07-15T06:59:17.716+0000 7fb55ea00640 -1 log_channel(cluster) log [ERR] : Health detail: HEALTH_ERR 8 osds(s) are not reachable 2024-07-15T06:59:18.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:17 smithi089 bash[79810]: debug 2024-07-15T06:59:17.716+0000 7fb55ea00640 -1 log_channel(cluster) log [ERR] : [ERR] OSD_UNREACHABLE: 8 osds(s) are not reachable 2024-07-15T06:59:18.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:17 smithi089 bash[79810]: debug 2024-07-15T06:59:17.716+0000 7fb55ea00640 -1 log_channel(cluster) log [ERR] : osd.0's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:59:18.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:17 smithi089 bash[79810]: debug 2024-07-15T06:59:17.716+0000 7fb55ea00640 -1 log_channel(cluster) log [ERR] : osd.1's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:59:18.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:17 smithi089 bash[79810]: debug 2024-07-15T06:59:17.716+0000 7fb55ea00640 -1 log_channel(cluster) log [ERR] : osd.2's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:59:18.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:17 smithi089 bash[79810]: debug 2024-07-15T06:59:17.716+0000 7fb55ea00640 -1 log_channel(cluster) log [ERR] : osd.3's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:59:18.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:17 smithi089 bash[79810]: debug 2024-07-15T06:59:17.716+0000 7fb55ea00640 -1 log_channel(cluster) log [ERR] : osd.4's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:59:18.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:17 smithi089 bash[79810]: debug 2024-07-15T06:59:17.716+0000 7fb55ea00640 -1 log_channel(cluster) log [ERR] : osd.5's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:59:18.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:17 smithi089 bash[79810]: debug 2024-07-15T06:59:17.716+0000 7fb55ea00640 -1 log_channel(cluster) log [ERR] : osd.6's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:59:18.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:17 smithi089 bash[79810]: debug 2024-07-15T06:59:17.716+0000 7fb55ea00640 -1 log_channel(cluster) log [ERR] : osd.7's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:59:19.054 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:59:18 smithi134 bash[60729]: ts=2024-07-15T06:59:18.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1102-g9a202024 (9a2020246f85463d4fdb115c94aa40b463caa1d2) quincy (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1102-g9a202024 (9a2020246f85463d4fdb115c94aa40b463caa1d2) quincy (stable)\", cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:59:21.869 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:59:21 smithi089 bash[72894]: debug 2024-07-15T06:59:21.392+0000 7f112575c640 -1 mgr.server handle_report got status from non-daemon mon.b 2024-07-15T06:59:22.804 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:59:22 smithi134 bash[60729]: ts=2024-07-15T06:59:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:59:22.837 INFO:teuthology.orchestra.run.smithi089.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:badc66b92d2a2cfbbce7f293b5accdc50f486d2b2d966876e46fe28d05f464f1 2024-07-15T06:59:23.111 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:22 smithi089 bash[79810]: debug 2024-07-15T06:59:22.808+0000 7fb55ea00640 -1 log_channel(cluster) log [ERR] : Health detail: HEALTH_ERR 1/3 mons down, quorum a,b; 8 osds(s) are not reachable 2024-07-15T06:59:23.111 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:22 smithi089 bash[79810]: debug 2024-07-15T06:59:22.808+0000 7fb55ea00640 -1 log_channel(cluster) log [ERR] : [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2024-07-15T06:59:23.111 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:22 smithi089 bash[79810]: debug 2024-07-15T06:59:22.808+0000 7fb55ea00640 -1 log_channel(cluster) log [ERR] : mon.c (rank 2) addr [v2:172.21.15.89:3301/0,v1:172.21.15.89:6790/0] is down (out of quorum) 2024-07-15T06:59:23.111 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:22 smithi089 bash[79810]: debug 2024-07-15T06:59:22.808+0000 7fb55ea00640 -1 log_channel(cluster) log [ERR] : [ERR] OSD_UNREACHABLE: 8 osds(s) are not reachable 2024-07-15T06:59:23.111 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:22 smithi089 bash[79810]: debug 2024-07-15T06:59:22.808+0000 7fb55ea00640 -1 log_channel(cluster) log [ERR] : osd.0's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:59:23.111 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:22 smithi089 bash[79810]: debug 2024-07-15T06:59:22.808+0000 7fb55ea00640 -1 log_channel(cluster) log [ERR] : osd.1's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:59:23.111 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:22 smithi089 bash[79810]: debug 2024-07-15T06:59:22.808+0000 7fb55ea00640 -1 log_channel(cluster) log [ERR] : osd.2's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:59:23.111 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:22 smithi089 bash[79810]: debug 2024-07-15T06:59:22.808+0000 7fb55ea00640 -1 log_channel(cluster) log [ERR] : osd.3's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:59:23.111 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:22 smithi089 bash[79810]: debug 2024-07-15T06:59:22.808+0000 7fb55ea00640 -1 log_channel(cluster) log [ERR] : osd.4's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:59:23.111 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:22 smithi089 bash[79810]: debug 2024-07-15T06:59:22.808+0000 7fb55ea00640 -1 log_channel(cluster) log [ERR] : osd.5's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:59:23.111 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:22 smithi089 bash[79810]: debug 2024-07-15T06:59:22.808+0000 7fb55ea00640 -1 log_channel(cluster) log [ERR] : osd.6's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:59:23.111 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:22 smithi089 bash[79810]: debug 2024-07-15T06:59:22.812+0000 7fb55ea00640 -1 log_channel(cluster) log [ERR] : osd.7's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:59:23.112 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:22 smithi089 bash[79810]: audit 2024-07-15T06:59:16.928521+0000 mgr.y (mgr.25244) 93 : audit [DBG] from='client.34179 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T06:59:23.112 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:22 smithi089 bash[79810]: cephadm 2024-07-15T06:59:16.929174+0000 mgr.y (mgr.25244) 94 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:badc66b92d2a2cfbbce7f293b5accdc50f486d2b2d966876e46fe28d05f464f1 2024-07-15T06:59:23.112 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:22 smithi089 bash[79810]: audit 2024-07-15T06:59:17.749914+0000 mon.a (mon.0) 130 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-07-15T06:59:23.112 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:22 smithi089 bash[79810]: cluster 2024-07-15T06:59:17.750238+0000 mon.a (mon.0) 131 : cluster [INF] mon.a calling monitor election 2024-07-15T06:59:23.112 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:22 smithi089 bash[79810]: cluster 2024-07-15T06:59:17.750911+0000 mon.b (mon.1) 2 : cluster [INF] mon.b calling monitor election 2024-07-15T06:59:23.112 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:22 smithi089 bash[79810]: audit 2024-07-15T06:59:17.773122+0000 mon.a (mon.0) 132 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-15T06:59:23.112 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:22 smithi089 bash[79810]: audit 2024-07-15T06:59:17.773590+0000 mon.a (mon.0) 133 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-15T06:59:23.113 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:22 smithi089 bash[79810]: cluster 2024-07-15T06:59:18.750110+0000 mgr.y (mgr.25244) 95 : cluster [DBG] pgmap v70: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:59:23.113 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:22 smithi089 bash[79810]: audit 2024-07-15T06:59:19.786464+0000 mon.a (mon.0) 134 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:59:23.113 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:22 smithi089 bash[79810]: cluster 2024-07-15T06:59:20.751132+0000 mgr.y (mgr.25244) 96 : cluster [DBG] pgmap v71: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:59:23.113 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:22 smithi089 bash[79810]: cluster 2024-07-15T06:59:22.752041+0000 mgr.y (mgr.25244) 97 : cluster [DBG] pgmap v72: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:59:23.113 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:22 smithi089 bash[79810]: cluster 2024-07-15T06:59:22.772684+0000 mon.a (mon.0) 135 : cluster [INF] mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-07-15T06:59:23.113 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:22 smithi089 bash[79810]: cluster 2024-07-15T06:59:22.780634+0000 mon.a (mon.0) 136 : cluster [DBG] monmap e4: 3 mons at {a=[v2:172.21.15.89:3300/0,v1:172.21.15.89:6789/0],b=[v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0],c=[v2:172.21.15.89:3301/0,v1:172.21.15.89:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-07-15T06:59:23.113 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:22 smithi089 bash[79810]: cluster 2024-07-15T06:59:22.800260+0000 mon.a (mon.0) 137 : cluster [DBG] fsmap foo:1 {0=foo.smithi134.wugtzh=up:active} 1 up:standby 2024-07-15T06:59:23.113 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:22 smithi089 bash[79810]: cluster 2024-07-15T06:59:22.800321+0000 mon.a (mon.0) 138 : cluster [DBG] osdmap e79: 8 total, 8 up, 8 in 2024-07-15T06:59:23.113 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:22 smithi089 bash[79810]: cluster 2024-07-15T06:59:22.800848+0000 mon.a (mon.0) 139 : cluster [DBG] mgrmap e40: y(active, since 2m), standbys: x 2024-07-15T06:59:23.113 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:22 smithi089 bash[79810]: cluster 2024-07-15T06:59:22.801169+0000 mon.a (mon.0) 140 : cluster [WRN] Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2024-07-15T06:59:23.113 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:22 smithi089 bash[79810]: cluster 2024-07-15T06:59:22.814774+0000 mon.a (mon.0) 141 : cluster [ERR] Health detail: HEALTH_ERR 1/3 mons down, quorum a,b; 8 osds(s) are not reachable 2024-07-15T06:59:23.114 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:22 smithi089 bash[79810]: cluster 2024-07-15T06:59:22.814831+0000 mon.a (mon.0) 142 : cluster [ERR] [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2024-07-15T06:59:23.114 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:22 smithi089 bash[79810]: cluster 2024-07-15T06:59:22.814873+0000 mon.a (mon.0) 143 : cluster [ERR] mon.c (rank 2) addr [v2:172.21.15.89:3301/0,v1:172.21.15.89:6790/0] is down (out of quorum) 2024-07-15T06:59:23.114 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:22 smithi089 bash[79810]: cluster 2024-07-15T06:59:22.814926+0000 mon.a (mon.0) 144 : cluster [ERR] [ERR] OSD_UNREACHABLE: 8 osds(s) are not reachable 2024-07-15T06:59:23.114 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:22 smithi089 bash[79810]: cluster 2024-07-15T06:59:22.814971+0000 mon.a (mon.0) 145 : cluster [ERR] osd.0's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:59:23.114 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:22 smithi089 bash[79810]: cluster 2024-07-15T06:59:22.815013+0000 mon.a (mon.0) 146 : cluster [ERR] osd.1's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:59:23.114 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:22 smithi089 bash[79810]: cluster 2024-07-15T06:59:22.815053+0000 mon.a (mon.0) 147 : cluster [ERR] osd.2's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:59:23.114 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:22 smithi089 bash[79810]: cluster 2024-07-15T06:59:22.815128+0000 mon.a (mon.0) 148 : cluster [ERR] osd.3's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:59:23.114 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:22 smithi089 bash[79810]: cluster 2024-07-15T06:59:22.815169+0000 mon.a (mon.0) 149 : cluster [ERR] osd.4's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:59:23.114 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:22 smithi089 bash[79810]: cluster 2024-07-15T06:59:22.815209+0000 mon.a (mon.0) 150 : cluster [ERR] osd.5's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:59:23.114 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:22 smithi089 bash[79810]: cluster 2024-07-15T06:59:22.815263+0000 mon.a (mon.0) 151 : cluster [ERR] osd.6's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:59:23.114 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:22 smithi089 bash[79810]: cluster 2024-07-15T06:59:22.815318+0000 mon.a (mon.0) 152 : cluster [ERR] osd.7's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:59:23.114 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:22 smithi089 bash[79810]: audit 2024-07-15T06:59:22.824419+0000 mon.a (mon.0) 153 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:59:23.115 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:22 smithi089 bash[79810]: audit 2024-07-15T06:59:22.836174+0000 mon.a (mon.0) 154 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:59:23.115 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:22 smithi089 bash[79810]: audit 2024-07-15T06:59:22.847668+0000 mon.a (mon.0) 155 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:59:23.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:22 smithi134 bash[67624]: audit 2024-07-15T06:59:16.928521+0000 mgr.y (mgr.25244) 93 : audit [DBG] from='client.34179 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T06:59:23.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:22 smithi134 bash[67624]: cephadm 2024-07-15T06:59:16.929174+0000 mgr.y (mgr.25244) 94 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:badc66b92d2a2cfbbce7f293b5accdc50f486d2b2d966876e46fe28d05f464f1 2024-07-15T06:59:23.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:22 smithi134 bash[67624]: audit 2024-07-15T06:59:17.749914+0000 mon.a (mon.0) 130 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-07-15T06:59:23.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:22 smithi134 bash[67624]: cluster 2024-07-15T06:59:17.750238+0000 mon.a (mon.0) 131 : cluster [INF] mon.a calling monitor election 2024-07-15T06:59:23.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:22 smithi134 bash[67624]: cluster 2024-07-15T06:59:17.750911+0000 mon.b (mon.1) 2 : cluster [INF] mon.b calling monitor election 2024-07-15T06:59:23.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:22 smithi134 bash[67624]: audit 2024-07-15T06:59:17.773122+0000 mon.a (mon.0) 132 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-15T06:59:23.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:22 smithi134 bash[67624]: audit 2024-07-15T06:59:17.773590+0000 mon.a (mon.0) 133 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-15T06:59:23.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:22 smithi134 bash[67624]: cluster 2024-07-15T06:59:18.750110+0000 mgr.y (mgr.25244) 95 : cluster [DBG] pgmap v70: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:59:23.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:22 smithi134 bash[67624]: audit 2024-07-15T06:59:19.786464+0000 mon.a (mon.0) 134 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:59:23.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:22 smithi134 bash[67624]: cluster 2024-07-15T06:59:20.751132+0000 mgr.y (mgr.25244) 96 : cluster [DBG] pgmap v71: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:59:23.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:22 smithi134 bash[67624]: cluster 2024-07-15T06:59:22.752041+0000 mgr.y (mgr.25244) 97 : cluster [DBG] pgmap v72: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:59:23.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:22 smithi134 bash[67624]: cluster 2024-07-15T06:59:22.772684+0000 mon.a (mon.0) 135 : cluster [INF] mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-07-15T06:59:23.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:22 smithi134 bash[67624]: cluster 2024-07-15T06:59:22.780634+0000 mon.a (mon.0) 136 : cluster [DBG] monmap e4: 3 mons at {a=[v2:172.21.15.89:3300/0,v1:172.21.15.89:6789/0],b=[v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0],c=[v2:172.21.15.89:3301/0,v1:172.21.15.89:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-07-15T06:59:23.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:22 smithi134 bash[67624]: cluster 2024-07-15T06:59:22.800260+0000 mon.a (mon.0) 137 : cluster [DBG] fsmap foo:1 {0=foo.smithi134.wugtzh=up:active} 1 up:standby 2024-07-15T06:59:23.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:22 smithi134 bash[67624]: cluster 2024-07-15T06:59:22.800321+0000 mon.a (mon.0) 138 : cluster [DBG] osdmap e79: 8 total, 8 up, 8 in 2024-07-15T06:59:23.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:22 smithi134 bash[67624]: cluster 2024-07-15T06:59:22.800848+0000 mon.a (mon.0) 139 : cluster [DBG] mgrmap e40: y(active, since 2m), standbys: x 2024-07-15T06:59:23.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:22 smithi134 bash[67624]: cluster 2024-07-15T06:59:22.801169+0000 mon.a (mon.0) 140 : cluster [WRN] Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2024-07-15T06:59:23.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:22 smithi134 bash[67624]: cluster 2024-07-15T06:59:22.814774+0000 mon.a (mon.0) 141 : cluster [ERR] Health detail: HEALTH_ERR 1/3 mons down, quorum a,b; 8 osds(s) are not reachable 2024-07-15T06:59:23.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:22 smithi134 bash[67624]: cluster 2024-07-15T06:59:22.814831+0000 mon.a (mon.0) 142 : cluster [ERR] [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2024-07-15T06:59:23.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:22 smithi134 bash[67624]: cluster 2024-07-15T06:59:22.814873+0000 mon.a (mon.0) 143 : cluster [ERR] mon.c (rank 2) addr [v2:172.21.15.89:3301/0,v1:172.21.15.89:6790/0] is down (out of quorum) 2024-07-15T06:59:23.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:22 smithi134 bash[67624]: cluster 2024-07-15T06:59:22.814926+0000 mon.a (mon.0) 144 : cluster [ERR] [ERR] OSD_UNREACHABLE: 8 osds(s) are not reachable 2024-07-15T06:59:23.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:22 smithi134 bash[67624]: cluster 2024-07-15T06:59:22.814971+0000 mon.a (mon.0) 145 : cluster [ERR] osd.0's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:59:23.307 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:22 smithi134 bash[67624]: cluster 2024-07-15T06:59:22.815013+0000 mon.a (mon.0) 146 : cluster [ERR] osd.1's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:59:23.307 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:22 smithi134 bash[67624]: cluster 2024-07-15T06:59:22.815053+0000 mon.a (mon.0) 147 : cluster [ERR] osd.2's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:59:23.307 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:22 smithi134 bash[67624]: cluster 2024-07-15T06:59:22.815128+0000 mon.a (mon.0) 148 : cluster [ERR] osd.3's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:59:23.307 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:22 smithi134 bash[67624]: cluster 2024-07-15T06:59:22.815169+0000 mon.a (mon.0) 149 : cluster [ERR] osd.4's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:59:23.307 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:22 smithi134 bash[67624]: cluster 2024-07-15T06:59:22.815209+0000 mon.a (mon.0) 150 : cluster [ERR] osd.5's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:59:23.307 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:22 smithi134 bash[67624]: cluster 2024-07-15T06:59:22.815263+0000 mon.a (mon.0) 151 : cluster [ERR] osd.6's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:59:23.307 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:22 smithi134 bash[67624]: cluster 2024-07-15T06:59:22.815318+0000 mon.a (mon.0) 152 : cluster [ERR] osd.7's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:59:23.307 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:22 smithi134 bash[67624]: audit 2024-07-15T06:59:22.824419+0000 mon.a (mon.0) 153 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:59:23.307 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:22 smithi134 bash[67624]: audit 2024-07-15T06:59:22.836174+0000 mon.a (mon.0) 154 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:59:23.307 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:22 smithi134 bash[67624]: audit 2024-07-15T06:59:22.847668+0000 mon.a (mon.0) 155 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:59:23.690 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -e sha1=6ea7aaa9f93d34dc4d21862afe00140dfddbd706 -- bash -c 'sleep 60' 2024-07-15T06:59:23.804 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:59:23 smithi134 bash[60729]: ts=2024-07-15T06:59:23.504Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi134\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi134\", job=\"node\", machine=\"x86_64\", nodename=\"smithi134\", release=\"5.15.0-113-generic\", sysname=\"Linux\", version=\"#123-Ubuntu SMP Mon Jun 10 08:16:17 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", domainname=\"(none)\", instance=\"smithi134\", job=\"node\", machine=\"x86_64\", nodename=\"smithi134\", release=\"5.15.0-113-generic\", sysname=\"Linux\", version=\"#123-Ubuntu SMP Mon Jun 10 08:16:17 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:59:24.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:23 smithi089 bash[79810]: debug 2024-07-15T06:59:23.880+0000 7fb55ea00640 -1 log_channel(cluster) log [ERR] : Health detail: HEALTH_ERR 8 osds(s) are not reachable 2024-07-15T06:59:24.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:23 smithi089 bash[79810]: debug 2024-07-15T06:59:23.880+0000 7fb55ea00640 -1 log_channel(cluster) log [ERR] : [ERR] OSD_UNREACHABLE: 8 osds(s) are not reachable 2024-07-15T06:59:24.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:23 smithi089 bash[79810]: debug 2024-07-15T06:59:23.880+0000 7fb55ea00640 -1 log_channel(cluster) log [ERR] : osd.0's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:59:24.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:23 smithi089 bash[79810]: debug 2024-07-15T06:59:23.880+0000 7fb55ea00640 -1 log_channel(cluster) log [ERR] : osd.1's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:59:24.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:23 smithi089 bash[79810]: debug 2024-07-15T06:59:23.880+0000 7fb55ea00640 -1 log_channel(cluster) log [ERR] : osd.2's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:59:24.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:23 smithi089 bash[79810]: debug 2024-07-15T06:59:23.880+0000 7fb55ea00640 -1 log_channel(cluster) log [ERR] : osd.3's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:59:24.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:23 smithi089 bash[79810]: debug 2024-07-15T06:59:23.880+0000 7fb55ea00640 -1 log_channel(cluster) log [ERR] : osd.4's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:59:24.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:23 smithi089 bash[79810]: debug 2024-07-15T06:59:23.880+0000 7fb55ea00640 -1 log_channel(cluster) log [ERR] : osd.5's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:59:24.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:23 smithi089 bash[79810]: debug 2024-07-15T06:59:23.880+0000 7fb55ea00640 -1 log_channel(cluster) log [ERR] : osd.6's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:59:24.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:23 smithi089 bash[79810]: debug 2024-07-15T06:59:23.880+0000 7fb55ea00640 -1 log_channel(cluster) log [ERR] : osd.7's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:59:24.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:23 smithi089 bash[82202]: audit 2024-07-15T06:59:16.928521+0000 mgr.y (mgr.25244) 93 : audit [DBG] from='client.34179 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T06:59:24.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:23 smithi089 bash[82202]: cephadm 2024-07-15T06:59:16.929174+0000 mgr.y (mgr.25244) 94 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:badc66b92d2a2cfbbce7f293b5accdc50f486d2b2d966876e46fe28d05f464f1 2024-07-15T06:59:24.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:23 smithi089 bash[82202]: audit 2024-07-15T06:59:17.749914+0000 mon.a (mon.0) 130 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-07-15T06:59:24.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:23 smithi089 bash[82202]: cluster 2024-07-15T06:59:17.750238+0000 mon.a (mon.0) 131 : cluster [INF] mon.a calling monitor election 2024-07-15T06:59:24.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:23 smithi089 bash[82202]: cluster 2024-07-15T06:59:17.750911+0000 mon.b (mon.1) 2 : cluster [INF] mon.b calling monitor election 2024-07-15T06:59:24.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:23 smithi089 bash[82202]: audit 2024-07-15T06:59:17.773122+0000 mon.a (mon.0) 132 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-15T06:59:24.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:23 smithi089 bash[82202]: audit 2024-07-15T06:59:17.773590+0000 mon.a (mon.0) 133 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-15T06:59:24.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:23 smithi089 bash[82202]: cluster 2024-07-15T06:59:18.750110+0000 mgr.y (mgr.25244) 95 : cluster [DBG] pgmap v70: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:59:24.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:23 smithi089 bash[82202]: audit 2024-07-15T06:59:19.786464+0000 mon.a (mon.0) 134 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:59:24.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:23 smithi089 bash[82202]: cluster 2024-07-15T06:59:20.751132+0000 mgr.y (mgr.25244) 96 : cluster [DBG] pgmap v71: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:59:24.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:23 smithi089 bash[82202]: cluster 2024-07-15T06:59:22.752041+0000 mgr.y (mgr.25244) 97 : cluster [DBG] pgmap v72: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:59:24.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:23 smithi089 bash[82202]: cluster 2024-07-15T06:59:22.772684+0000 mon.a (mon.0) 135 : cluster [INF] mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-07-15T06:59:24.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:23 smithi089 bash[82202]: cluster 2024-07-15T06:59:22.780634+0000 mon.a (mon.0) 136 : cluster [DBG] monmap e4: 3 mons at {a=[v2:172.21.15.89:3300/0,v1:172.21.15.89:6789/0],b=[v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0],c=[v2:172.21.15.89:3301/0,v1:172.21.15.89:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-07-15T06:59:24.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:23 smithi089 bash[82202]: cluster 2024-07-15T06:59:22.800260+0000 mon.a (mon.0) 137 : cluster [DBG] fsmap foo:1 {0=foo.smithi134.wugtzh=up:active} 1 up:standby 2024-07-15T06:59:24.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:23 smithi089 bash[82202]: cluster 2024-07-15T06:59:22.800321+0000 mon.a (mon.0) 138 : cluster [DBG] osdmap e79: 8 total, 8 up, 8 in 2024-07-15T06:59:24.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:23 smithi089 bash[82202]: cluster 2024-07-15T06:59:22.800848+0000 mon.a (mon.0) 139 : cluster [DBG] mgrmap e40: y(active, since 2m), standbys: x 2024-07-15T06:59:24.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:23 smithi089 bash[82202]: cluster 2024-07-15T06:59:22.801169+0000 mon.a (mon.0) 140 : cluster [WRN] Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2024-07-15T06:59:24.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:23 smithi089 bash[82202]: cluster 2024-07-15T06:59:22.814774+0000 mon.a (mon.0) 141 : cluster [ERR] Health detail: HEALTH_ERR 1/3 mons down, quorum a,b; 8 osds(s) are not reachable 2024-07-15T06:59:24.123 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:23 smithi089 bash[82202]: cluster 2024-07-15T06:59:22.814831+0000 mon.a (mon.0) 142 : cluster [ERR] [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2024-07-15T06:59:24.123 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:23 smithi089 bash[82202]: cluster 2024-07-15T06:59:22.814873+0000 mon.a (mon.0) 143 : cluster [ERR] mon.c (rank 2) addr [v2:172.21.15.89:3301/0,v1:172.21.15.89:6790/0] is down (out of quorum) 2024-07-15T06:59:24.123 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:23 smithi089 bash[82202]: cluster 2024-07-15T06:59:22.814926+0000 mon.a (mon.0) 144 : cluster [ERR] [ERR] OSD_UNREACHABLE: 8 osds(s) are not reachable 2024-07-15T06:59:24.123 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:23 smithi089 bash[82202]: cluster 2024-07-15T06:59:22.814971+0000 mon.a (mon.0) 145 : cluster [ERR] osd.0's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:59:24.123 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:23 smithi089 bash[82202]: cluster 2024-07-15T06:59:22.815013+0000 mon.a (mon.0) 146 : cluster [ERR] osd.1's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:59:24.123 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:23 smithi089 bash[82202]: cluster 2024-07-15T06:59:22.815053+0000 mon.a (mon.0) 147 : cluster [ERR] osd.2's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:59:24.123 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:23 smithi089 bash[82202]: cluster 2024-07-15T06:59:22.815128+0000 mon.a (mon.0) 148 : cluster [ERR] osd.3's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:59:24.123 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:23 smithi089 bash[82202]: cluster 2024-07-15T06:59:22.815169+0000 mon.a (mon.0) 149 : cluster [ERR] osd.4's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:59:24.123 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:23 smithi089 bash[82202]: cluster 2024-07-15T06:59:22.815209+0000 mon.a (mon.0) 150 : cluster [ERR] osd.5's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:59:24.123 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:23 smithi089 bash[82202]: cluster 2024-07-15T06:59:22.815263+0000 mon.a (mon.0) 151 : cluster [ERR] osd.6's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:59:24.123 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:23 smithi089 bash[82202]: cluster 2024-07-15T06:59:22.815318+0000 mon.a (mon.0) 152 : cluster [ERR] osd.7's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:59:24.123 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:23 smithi089 bash[82202]: audit 2024-07-15T06:59:22.824419+0000 mon.a (mon.0) 153 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:59:24.123 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:23 smithi089 bash[82202]: audit 2024-07-15T06:59:22.836174+0000 mon.a (mon.0) 154 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:59:24.123 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:23 smithi089 bash[82202]: audit 2024-07-15T06:59:22.847668+0000 mon.a (mon.0) 155 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:59:24.554 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:59:24 smithi134 bash[60729]: ts=2024-07-15T06:59:24.454Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:59:24.555 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:59:24 smithi134 bash[60729]: ts=2024-07-15T06:59:24.455Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:59:24.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:24 smithi134 bash[67624]: cluster 2024-07-15T06:59:17.772708+0000 mon.c (mon.2) 2 : cluster [INF] mon.c calling monitor election 2024-07-15T06:59:24.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:24 smithi134 bash[67624]: cluster 2024-07-15T06:59:23.809399+0000 mon.a (mon.0) 159 : cluster [INF] mon.a calling monitor election 2024-07-15T06:59:24.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:24 smithi134 bash[67624]: cluster 2024-07-15T06:59:23.809993+0000 mon.b (mon.1) 3 : cluster [INF] mon.b calling monitor election 2024-07-15T06:59:24.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:24 smithi134 bash[67624]: cluster 2024-07-15T06:59:23.850575+0000 mon.a (mon.0) 160 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-07-15T06:59:24.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:24 smithi134 bash[67624]: cluster 2024-07-15T06:59:23.856106+0000 mon.a (mon.0) 161 : cluster [DBG] monmap e4: 3 mons at {a=[v2:172.21.15.89:3300/0,v1:172.21.15.89:6789/0],b=[v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0],c=[v2:172.21.15.89:3301/0,v1:172.21.15.89:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-07-15T06:59:24.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:24 smithi134 bash[67624]: cluster 2024-07-15T06:59:23.856819+0000 mon.a (mon.0) 162 : cluster [DBG] fsmap foo:1 {0=foo.smithi134.wugtzh=up:active} 1 up:standby 2024-07-15T06:59:24.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:24 smithi134 bash[67624]: cluster 2024-07-15T06:59:23.856858+0000 mon.a (mon.0) 163 : cluster [DBG] osdmap e79: 8 total, 8 up, 8 in 2024-07-15T06:59:24.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:24 smithi134 bash[67624]: cluster 2024-07-15T06:59:23.857337+0000 mon.a (mon.0) 164 : cluster [DBG] mgrmap e40: y(active, since 2m), standbys: x 2024-07-15T06:59:24.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:24 smithi134 bash[67624]: cluster 2024-07-15T06:59:23.857529+0000 mon.a (mon.0) 165 : cluster [INF] Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-07-15T06:59:24.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:24 smithi134 bash[67624]: cluster 2024-07-15T06:59:23.886718+0000 mon.a (mon.0) 166 : cluster [ERR] Health detail: HEALTH_ERR 8 osds(s) are not reachable 2024-07-15T06:59:24.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:24 smithi134 bash[67624]: cluster 2024-07-15T06:59:23.886752+0000 mon.a (mon.0) 167 : cluster [ERR] [ERR] OSD_UNREACHABLE: 8 osds(s) are not reachable 2024-07-15T06:59:24.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:24 smithi134 bash[67624]: cluster 2024-07-15T06:59:23.886762+0000 mon.a (mon.0) 168 : cluster [ERR] osd.0's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:59:24.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:24 smithi134 bash[67624]: cluster 2024-07-15T06:59:23.886773+0000 mon.a (mon.0) 169 : cluster [ERR] osd.1's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:59:24.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:24 smithi134 bash[67624]: cluster 2024-07-15T06:59:23.886784+0000 mon.a (mon.0) 170 : cluster [ERR] osd.2's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:59:24.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:24 smithi134 bash[67624]: cluster 2024-07-15T06:59:23.886794+0000 mon.a (mon.0) 171 : cluster [ERR] osd.3's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:59:24.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:24 smithi134 bash[67624]: cluster 2024-07-15T06:59:23.886806+0000 mon.a (mon.0) 172 : cluster [ERR] osd.4's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:59:24.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:24 smithi134 bash[67624]: cluster 2024-07-15T06:59:23.886817+0000 mon.a (mon.0) 173 : cluster [ERR] osd.5's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:59:24.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:24 smithi134 bash[67624]: cluster 2024-07-15T06:59:23.886830+0000 mon.a (mon.0) 174 : cluster [ERR] osd.6's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:59:24.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:24 smithi134 bash[67624]: cluster 2024-07-15T06:59:23.886841+0000 mon.a (mon.0) 175 : cluster [ERR] osd.7's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:59:24.619 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:59:24 smithi089 bash[72894]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:59:24] "GET /metrics HTTP/1.1" 200 33864 "" "Prometheus/2.43.0" 2024-07-15T06:59:24.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:24 smithi089 bash[79810]: cluster 2024-07-15T06:59:17.772708+0000 mon.c (mon.2) 2 : cluster [INF] mon.c calling monitor election 2024-07-15T06:59:24.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:24 smithi089 bash[79810]: cluster 2024-07-15T06:59:23.809399+0000 mon.a (mon.0) 159 : cluster [INF] mon.a calling monitor election 2024-07-15T06:59:24.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:24 smithi089 bash[79810]: cluster 2024-07-15T06:59:23.809993+0000 mon.b (mon.1) 3 : cluster [INF] mon.b calling monitor election 2024-07-15T06:59:24.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:24 smithi089 bash[79810]: cluster 2024-07-15T06:59:23.850575+0000 mon.a (mon.0) 160 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-07-15T06:59:24.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:24 smithi089 bash[79810]: cluster 2024-07-15T06:59:23.856106+0000 mon.a (mon.0) 161 : cluster [DBG] monmap e4: 3 mons at {a=[v2:172.21.15.89:3300/0,v1:172.21.15.89:6789/0],b=[v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0],c=[v2:172.21.15.89:3301/0,v1:172.21.15.89:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-07-15T06:59:24.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:24 smithi089 bash[79810]: cluster 2024-07-15T06:59:23.856819+0000 mon.a (mon.0) 162 : cluster [DBG] fsmap foo:1 {0=foo.smithi134.wugtzh=up:active} 1 up:standby 2024-07-15T06:59:24.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:24 smithi089 bash[79810]: cluster 2024-07-15T06:59:23.856858+0000 mon.a (mon.0) 163 : cluster [DBG] osdmap e79: 8 total, 8 up, 8 in 2024-07-15T06:59:24.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:24 smithi089 bash[79810]: cluster 2024-07-15T06:59:23.857337+0000 mon.a (mon.0) 164 : cluster [DBG] mgrmap e40: y(active, since 2m), standbys: x 2024-07-15T06:59:24.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:24 smithi089 bash[79810]: cluster 2024-07-15T06:59:23.857529+0000 mon.a (mon.0) 165 : cluster [INF] Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-07-15T06:59:24.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:24 smithi089 bash[79810]: cluster 2024-07-15T06:59:23.886718+0000 mon.a (mon.0) 166 : cluster [ERR] Health detail: HEALTH_ERR 8 osds(s) are not reachable 2024-07-15T06:59:24.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:24 smithi089 bash[79810]: cluster 2024-07-15T06:59:23.886752+0000 mon.a (mon.0) 167 : cluster [ERR] [ERR] OSD_UNREACHABLE: 8 osds(s) are not reachable 2024-07-15T06:59:24.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:24 smithi089 bash[79810]: cluster 2024-07-15T06:59:23.886762+0000 mon.a (mon.0) 168 : cluster [ERR] osd.0's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:59:24.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:24 smithi089 bash[79810]: cluster 2024-07-15T06:59:23.886773+0000 mon.a (mon.0) 169 : cluster [ERR] osd.1's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:59:24.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:24 smithi089 bash[79810]: cluster 2024-07-15T06:59:23.886784+0000 mon.a (mon.0) 170 : cluster [ERR] osd.2's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:59:24.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:24 smithi089 bash[79810]: cluster 2024-07-15T06:59:23.886794+0000 mon.a (mon.0) 171 : cluster [ERR] osd.3's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:59:24.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:24 smithi089 bash[79810]: cluster 2024-07-15T06:59:23.886806+0000 mon.a (mon.0) 172 : cluster [ERR] osd.4's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:59:24.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:24 smithi089 bash[79810]: cluster 2024-07-15T06:59:23.886817+0000 mon.a (mon.0) 173 : cluster [ERR] osd.5's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:59:24.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:24 smithi089 bash[79810]: cluster 2024-07-15T06:59:23.886830+0000 mon.a (mon.0) 174 : cluster [ERR] osd.6's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:59:24.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:24 smithi089 bash[79810]: cluster 2024-07-15T06:59:23.886841+0000 mon.a (mon.0) 175 : cluster [ERR] osd.7's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:59:24.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:24 smithi089 bash[82202]: cluster 2024-07-15T06:59:17.772708+0000 mon.c (mon.2) 2 : cluster [INF] mon.c calling monitor election 2024-07-15T06:59:24.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:24 smithi089 bash[82202]: cluster 2024-07-15T06:59:23.809399+0000 mon.a (mon.0) 159 : cluster [INF] mon.a calling monitor election 2024-07-15T06:59:24.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:24 smithi089 bash[82202]: cluster 2024-07-15T06:59:23.809993+0000 mon.b (mon.1) 3 : cluster [INF] mon.b calling monitor election 2024-07-15T06:59:24.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:24 smithi089 bash[82202]: cluster 2024-07-15T06:59:23.850575+0000 mon.a (mon.0) 160 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-07-15T06:59:24.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:24 smithi089 bash[82202]: cluster 2024-07-15T06:59:23.856106+0000 mon.a (mon.0) 161 : cluster [DBG] monmap e4: 3 mons at {a=[v2:172.21.15.89:3300/0,v1:172.21.15.89:6789/0],b=[v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0],c=[v2:172.21.15.89:3301/0,v1:172.21.15.89:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-07-15T06:59:24.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:24 smithi089 bash[82202]: cluster 2024-07-15T06:59:23.856819+0000 mon.a (mon.0) 162 : cluster [DBG] fsmap foo:1 {0=foo.smithi134.wugtzh=up:active} 1 up:standby 2024-07-15T06:59:24.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:24 smithi089 bash[82202]: cluster 2024-07-15T06:59:23.856858+0000 mon.a (mon.0) 163 : cluster [DBG] osdmap e79: 8 total, 8 up, 8 in 2024-07-15T06:59:24.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:24 smithi089 bash[82202]: cluster 2024-07-15T06:59:23.857337+0000 mon.a (mon.0) 164 : cluster [DBG] mgrmap e40: y(active, since 2m), standbys: x 2024-07-15T06:59:24.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:24 smithi089 bash[82202]: cluster 2024-07-15T06:59:23.857529+0000 mon.a (mon.0) 165 : cluster [INF] Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-07-15T06:59:24.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:24 smithi089 bash[82202]: cluster 2024-07-15T06:59:23.886718+0000 mon.a (mon.0) 166 : cluster [ERR] Health detail: HEALTH_ERR 8 osds(s) are not reachable 2024-07-15T06:59:24.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:24 smithi089 bash[82202]: cluster 2024-07-15T06:59:23.886752+0000 mon.a (mon.0) 167 : cluster [ERR] [ERR] OSD_UNREACHABLE: 8 osds(s) are not reachable 2024-07-15T06:59:24.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:24 smithi089 bash[82202]: cluster 2024-07-15T06:59:23.886762+0000 mon.a (mon.0) 168 : cluster [ERR] osd.0's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:59:24.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:24 smithi089 bash[82202]: cluster 2024-07-15T06:59:23.886773+0000 mon.a (mon.0) 169 : cluster [ERR] osd.1's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:59:24.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:24 smithi089 bash[82202]: cluster 2024-07-15T06:59:23.886784+0000 mon.a (mon.0) 170 : cluster [ERR] osd.2's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:59:24.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:24 smithi089 bash[82202]: cluster 2024-07-15T06:59:23.886794+0000 mon.a (mon.0) 171 : cluster [ERR] osd.3's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:59:24.623 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:24 smithi089 bash[82202]: cluster 2024-07-15T06:59:23.886806+0000 mon.a (mon.0) 172 : cluster [ERR] osd.4's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:59:24.623 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:24 smithi089 bash[82202]: cluster 2024-07-15T06:59:23.886817+0000 mon.a (mon.0) 173 : cluster [ERR] osd.5's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:59:24.623 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:24 smithi089 bash[82202]: cluster 2024-07-15T06:59:23.886830+0000 mon.a (mon.0) 174 : cluster [ERR] osd.6's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:59:24.623 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:24 smithi089 bash[82202]: cluster 2024-07-15T06:59:23.886841+0000 mon.a (mon.0) 175 : cluster [ERR] osd.7's public address is not in '172.21.0.1/32,172.21.0.2/32,172.21.0.0/20,172.21.15.254/32' subnet 2024-07-15T06:59:25.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:25 smithi134 bash[67624]: cluster 2024-07-15T06:59:24.752618+0000 mgr.y (mgr.25244) 98 : cluster [DBG] pgmap v73: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:59:25.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:25 smithi089 bash[79810]: cluster 2024-07-15T06:59:24.752618+0000 mgr.y (mgr.25244) 98 : cluster [DBG] pgmap v73: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:59:25.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:25 smithi089 bash[82202]: cluster 2024-07-15T06:59:24.752618+0000 mgr.y (mgr.25244) 98 : cluster [DBG] pgmap v73: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:59:27.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:27 smithi134 bash[67624]: cluster 2024-07-15T06:59:26.753735+0000 mgr.y (mgr.25244) 99 : cluster [DBG] pgmap v74: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:59:27.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:27 smithi089 bash[79810]: cluster 2024-07-15T06:59:26.753735+0000 mgr.y (mgr.25244) 99 : cluster [DBG] pgmap v74: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:59:27.870 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:27 smithi089 bash[82202]: cluster 2024-07-15T06:59:26.753735+0000 mgr.y (mgr.25244) 99 : cluster [DBG] pgmap v74: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:59:28.480 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T06:59:29.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:28 smithi134 bash[67624]: cluster 2024-07-15T06:59:28.754428+0000 mgr.y (mgr.25244) 100 : cluster [DBG] pgmap v75: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:59:29.054 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:59:28 smithi134 bash[60729]: ts=2024-07-15T06:59:28.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1102-g9a202024 (9a2020246f85463d4fdb115c94aa40b463caa1d2) quincy (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1102-g9a202024 (9a2020246f85463d4fdb115c94aa40b463caa1d2) quincy (stable)\", cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:59:29.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:28 smithi089 bash[79810]: cluster 2024-07-15T06:59:28.754428+0000 mgr.y (mgr.25244) 100 : cluster [DBG] pgmap v75: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:59:29.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:28 smithi089 bash[82202]: cluster 2024-07-15T06:59:28.754428+0000 mgr.y (mgr.25244) 100 : cluster [DBG] pgmap v75: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:59:30.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:30 smithi089 bash[79810]: audit 2024-07-15T06:59:29.014818+0000 mon.a (mon.0) 176 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:59:30.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:30 smithi089 bash[79810]: audit 2024-07-15T06:59:29.024869+0000 mon.a (mon.0) 177 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:59:30.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:30 smithi089 bash[79810]: audit 2024-07-15T06:59:29.805440+0000 mon.a (mon.0) 178 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:59:30.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:30 smithi089 bash[79810]: audit 2024-07-15T06:59:29.816514+0000 mon.a (mon.0) 179 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:59:30.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:30 smithi089 bash[82202]: audit 2024-07-15T06:59:29.014818+0000 mon.a (mon.0) 176 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:59:30.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:30 smithi089 bash[82202]: audit 2024-07-15T06:59:29.024869+0000 mon.a (mon.0) 177 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:59:30.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:30 smithi089 bash[82202]: audit 2024-07-15T06:59:29.805440+0000 mon.a (mon.0) 178 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:59:30.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:30 smithi089 bash[82202]: audit 2024-07-15T06:59:29.816514+0000 mon.a (mon.0) 179 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:59:30.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:30 smithi134 bash[67624]: audit 2024-07-15T06:59:29.014818+0000 mon.a (mon.0) 176 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:59:30.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:30 smithi134 bash[67624]: audit 2024-07-15T06:59:29.024869+0000 mon.a (mon.0) 177 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:59:30.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:30 smithi134 bash[67624]: audit 2024-07-15T06:59:29.805440+0000 mon.a (mon.0) 178 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:59:30.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:30 smithi134 bash[67624]: audit 2024-07-15T06:59:29.816514+0000 mon.a (mon.0) 179 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:59:31.123 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:31 smithi089 bash[82202]: cluster 2024-07-15T06:59:30.755049+0000 mgr.y (mgr.25244) 101 : cluster [DBG] pgmap v76: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:59:31.123 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:31 smithi089 bash[79810]: cluster 2024-07-15T06:59:30.755049+0000 mgr.y (mgr.25244) 101 : cluster [DBG] pgmap v76: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:59:31.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:31 smithi134 bash[67624]: cluster 2024-07-15T06:59:30.755049+0000 mgr.y (mgr.25244) 101 : cluster [DBG] pgmap v76: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:59:32.804 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:59:32 smithi134 bash[60729]: ts=2024-07-15T06:59:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:59:33.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:32 smithi089 bash[79810]: cluster 2024-07-15T06:59:32.755655+0000 mgr.y (mgr.25244) 102 : cluster [DBG] pgmap v77: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:59:33.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:32 smithi089 bash[82202]: cluster 2024-07-15T06:59:32.755655+0000 mgr.y (mgr.25244) 102 : cluster [DBG] pgmap v77: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:59:33.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:32 smithi134 bash[67624]: cluster 2024-07-15T06:59:32.755655+0000 mgr.y (mgr.25244) 102 : cluster [DBG] pgmap v77: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:59:33.804 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:59:33 smithi134 bash[60729]: ts=2024-07-15T06:59:33.508Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi134\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi134\", job=\"node\", machine=\"x86_64\", nodename=\"smithi134\", release=\"5.15.0-113-generic\", sysname=\"Linux\", version=\"#123-Ubuntu SMP Mon Jun 10 08:16:17 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", domainname=\"(none)\", instance=\"smithi134\", job=\"node\", machine=\"x86_64\", nodename=\"smithi134\", release=\"5.15.0-113-generic\", sysname=\"Linux\", version=\"#123-Ubuntu SMP Mon Jun 10 08:16:17 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:59:34.619 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:59:34 smithi089 bash[72894]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:59:34] "GET /metrics HTTP/1.1" 200 33864 "" "Prometheus/2.43.0" 2024-07-15T06:59:34.790 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:59:34 smithi134 bash[60729]: ts=2024-07-15T06:59:34.453Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:59:34.790 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:59:34 smithi134 bash[60729]: ts=2024-07-15T06:59:34.454Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:59:35.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:34 smithi134 bash[67624]: cluster 2024-07-15T06:59:34.756231+0000 mgr.y (mgr.25244) 103 : cluster [DBG] pgmap v78: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:59:35.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:34 smithi134 bash[67624]: audit 2024-07-15T06:59:34.786837+0000 mon.a (mon.0) 180 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:59:35.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:34 smithi089 bash[79810]: cluster 2024-07-15T06:59:34.756231+0000 mgr.y (mgr.25244) 103 : cluster [DBG] pgmap v78: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:59:35.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:34 smithi089 bash[79810]: audit 2024-07-15T06:59:34.786837+0000 mon.a (mon.0) 180 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:59:35.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:34 smithi089 bash[82202]: cluster 2024-07-15T06:59:34.756231+0000 mgr.y (mgr.25244) 103 : cluster [DBG] pgmap v78: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:59:35.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:34 smithi089 bash[82202]: audit 2024-07-15T06:59:34.786837+0000 mon.a (mon.0) 180 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:59:37.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:36 smithi089 bash[79810]: cluster 2024-07-15T06:59:36.757289+0000 mgr.y (mgr.25244) 104 : cluster [DBG] pgmap v79: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:59:37.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:36 smithi089 bash[82202]: cluster 2024-07-15T06:59:36.757289+0000 mgr.y (mgr.25244) 104 : cluster [DBG] pgmap v79: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:59:37.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:36 smithi134 bash[67624]: cluster 2024-07-15T06:59:36.757289+0000 mgr.y (mgr.25244) 104 : cluster [DBG] pgmap v79: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:59:38.907 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:59:38 smithi134 bash[60729]: ts=2024-07-15T06:59:38.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1102-g9a202024 (9a2020246f85463d4fdb115c94aa40b463caa1d2) quincy (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1102-g9a202024 (9a2020246f85463d4fdb115c94aa40b463caa1d2) quincy (stable)\", cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:59:38.908 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:38 smithi134 bash[67624]: cluster 2024-07-15T06:59:38.757919+0000 mgr.y (mgr.25244) 105 : cluster [DBG] pgmap v80: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:59:39.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:38 smithi089 bash[79810]: cluster 2024-07-15T06:59:38.757919+0000 mgr.y (mgr.25244) 105 : cluster [DBG] pgmap v80: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:59:39.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:38 smithi089 bash[82202]: cluster 2024-07-15T06:59:38.757919+0000 mgr.y (mgr.25244) 105 : cluster [DBG] pgmap v80: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:59:41.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:40 smithi134 bash[67624]: audit 2024-07-15T06:59:39.649737+0000 mon.a (mon.0) 181 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:59:41.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:40 smithi134 bash[67624]: audit 2024-07-15T06:59:39.660117+0000 mon.a (mon.0) 182 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:59:41.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:40 smithi134 bash[67624]: audit 2024-07-15T06:59:39.662360+0000 mon.a (mon.0) 183 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:59:41.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:40 smithi134 bash[67624]: audit 2024-07-15T06:59:39.664073+0000 mon.a (mon.0) 184 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:59:41.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:40 smithi134 bash[67624]: audit 2024-07-15T06:59:39.674475+0000 mon.a (mon.0) 185 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:59:41.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:40 smithi134 bash[67624]: cephadm 2024-07-15T06:59:39.739835+0000 mgr.y (mgr.25244) 106 : cephadm [INF] Reconfiguring mon.a (monmap changed)... 2024-07-15T06:59:41.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:40 smithi134 bash[67624]: audit 2024-07-15T06:59:39.740343+0000 mon.a (mon.0) 186 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-07-15T06:59:41.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:40 smithi134 bash[67624]: audit 2024-07-15T06:59:39.741758+0000 mon.a (mon.0) 187 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-07-15T06:59:41.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:40 smithi134 bash[67624]: audit 2024-07-15T06:59:39.743088+0000 mon.a (mon.0) 188 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:59:41.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:40 smithi134 bash[67624]: cephadm 2024-07-15T06:59:39.744694+0000 mgr.y (mgr.25244) 107 : cephadm [INF] Reconfiguring daemon mon.a on smithi089 2024-07-15T06:59:41.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:40 smithi089 bash[79810]: audit 2024-07-15T06:59:39.649737+0000 mon.a (mon.0) 181 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:59:41.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:40 smithi089 bash[79810]: audit 2024-07-15T06:59:39.660117+0000 mon.a (mon.0) 182 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:59:41.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:40 smithi089 bash[79810]: audit 2024-07-15T06:59:39.662360+0000 mon.a (mon.0) 183 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:59:41.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:40 smithi089 bash[79810]: audit 2024-07-15T06:59:39.664073+0000 mon.a (mon.0) 184 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:59:41.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:40 smithi089 bash[79810]: audit 2024-07-15T06:59:39.674475+0000 mon.a (mon.0) 185 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:59:41.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:40 smithi089 bash[79810]: cephadm 2024-07-15T06:59:39.739835+0000 mgr.y (mgr.25244) 106 : cephadm [INF] Reconfiguring mon.a (monmap changed)... 2024-07-15T06:59:41.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:40 smithi089 bash[79810]: audit 2024-07-15T06:59:39.740343+0000 mon.a (mon.0) 186 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-07-15T06:59:41.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:40 smithi089 bash[79810]: audit 2024-07-15T06:59:39.741758+0000 mon.a (mon.0) 187 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-07-15T06:59:41.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:40 smithi089 bash[79810]: audit 2024-07-15T06:59:39.743088+0000 mon.a (mon.0) 188 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:59:41.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:40 smithi089 bash[79810]: cephadm 2024-07-15T06:59:39.744694+0000 mgr.y (mgr.25244) 107 : cephadm [INF] Reconfiguring daemon mon.a on smithi089 2024-07-15T06:59:41.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:40 smithi089 bash[82202]: audit 2024-07-15T06:59:39.649737+0000 mon.a (mon.0) 181 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:59:41.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:40 smithi089 bash[82202]: audit 2024-07-15T06:59:39.660117+0000 mon.a (mon.0) 182 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:59:41.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:40 smithi089 bash[82202]: audit 2024-07-15T06:59:39.662360+0000 mon.a (mon.0) 183 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:59:41.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:40 smithi089 bash[82202]: audit 2024-07-15T06:59:39.664073+0000 mon.a (mon.0) 184 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T06:59:41.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:40 smithi089 bash[82202]: audit 2024-07-15T06:59:39.674475+0000 mon.a (mon.0) 185 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:59:41.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:40 smithi089 bash[82202]: cephadm 2024-07-15T06:59:39.739835+0000 mgr.y (mgr.25244) 106 : cephadm [INF] Reconfiguring mon.a (monmap changed)... 2024-07-15T06:59:41.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:40 smithi089 bash[82202]: audit 2024-07-15T06:59:39.740343+0000 mon.a (mon.0) 186 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-07-15T06:59:41.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:40 smithi089 bash[82202]: audit 2024-07-15T06:59:39.741758+0000 mon.a (mon.0) 187 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-07-15T06:59:41.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:40 smithi089 bash[82202]: audit 2024-07-15T06:59:39.743088+0000 mon.a (mon.0) 188 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:59:41.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:40 smithi089 bash[82202]: cephadm 2024-07-15T06:59:39.744694+0000 mgr.y (mgr.25244) 107 : cephadm [INF] Reconfiguring daemon mon.a on smithi089 2024-07-15T06:59:42.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:41 smithi134 bash[67624]: cluster 2024-07-15T06:59:40.758954+0000 mgr.y (mgr.25244) 108 : cluster [DBG] pgmap v81: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:59:42.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:41 smithi089 bash[79810]: cluster 2024-07-15T06:59:40.758954+0000 mgr.y (mgr.25244) 108 : cluster [DBG] pgmap v81: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:59:42.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:41 smithi089 bash[82202]: cluster 2024-07-15T06:59:40.758954+0000 mgr.y (mgr.25244) 108 : cluster [DBG] pgmap v81: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:59:42.804 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:59:42 smithi134 bash[60729]: ts=2024-07-15T06:59:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:59:43.503 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:43 smithi134 bash[67624]: audit 2024-07-15T06:59:42.178646+0000 mon.a (mon.0) 189 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:59:43.503 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:43 smithi134 bash[67624]: audit 2024-07-15T06:59:42.187900+0000 mon.a (mon.0) 190 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:59:43.503 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:43 smithi134 bash[67624]: cephadm 2024-07-15T06:59:42.189568+0000 mgr.y (mgr.25244) 109 : cephadm [INF] Reconfiguring mgr.y (monmap changed)... 2024-07-15T06:59:43.504 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:43 smithi134 bash[67624]: audit 2024-07-15T06:59:42.190133+0000 mon.a (mon.0) 191 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-07-15T06:59:43.504 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:43 smithi134 bash[67624]: audit 2024-07-15T06:59:42.191816+0000 mon.a (mon.0) 192 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-07-15T06:59:43.504 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:43 smithi134 bash[67624]: audit 2024-07-15T06:59:42.193606+0000 mon.a (mon.0) 193 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:59:43.504 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:43 smithi134 bash[67624]: cephadm 2024-07-15T06:59:42.195191+0000 mgr.y (mgr.25244) 110 : cephadm [INF] Reconfiguring daemon mgr.y on smithi089 2024-07-15T06:59:43.504 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:43 smithi134 bash[67624]: cluster 2024-07-15T06:59:42.759596+0000 mgr.y (mgr.25244) 111 : cluster [DBG] pgmap v82: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:59:43.539 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:43 smithi089 bash[82202]: audit 2024-07-15T06:59:42.178646+0000 mon.a (mon.0) 189 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:59:43.539 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:43 smithi089 bash[82202]: audit 2024-07-15T06:59:42.187900+0000 mon.a (mon.0) 190 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:59:43.539 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:43 smithi089 bash[82202]: cephadm 2024-07-15T06:59:42.189568+0000 mgr.y (mgr.25244) 109 : cephadm [INF] Reconfiguring mgr.y (monmap changed)... 2024-07-15T06:59:43.539 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:43 smithi089 bash[82202]: audit 2024-07-15T06:59:42.190133+0000 mon.a (mon.0) 191 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-07-15T06:59:43.540 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:43 smithi089 bash[82202]: audit 2024-07-15T06:59:42.191816+0000 mon.a (mon.0) 192 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-07-15T06:59:43.540 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:43 smithi089 bash[82202]: audit 2024-07-15T06:59:42.193606+0000 mon.a (mon.0) 193 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:59:43.540 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:43 smithi089 bash[82202]: cephadm 2024-07-15T06:59:42.195191+0000 mgr.y (mgr.25244) 110 : cephadm [INF] Reconfiguring daemon mgr.y on smithi089 2024-07-15T06:59:43.540 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:43 smithi089 bash[82202]: cluster 2024-07-15T06:59:42.759596+0000 mgr.y (mgr.25244) 111 : cluster [DBG] pgmap v82: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:59:43.540 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:43 smithi089 bash[79810]: audit 2024-07-15T06:59:42.178646+0000 mon.a (mon.0) 189 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:59:43.540 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:43 smithi089 bash[79810]: audit 2024-07-15T06:59:42.187900+0000 mon.a (mon.0) 190 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:59:43.540 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:43 smithi089 bash[79810]: cephadm 2024-07-15T06:59:42.189568+0000 mgr.y (mgr.25244) 109 : cephadm [INF] Reconfiguring mgr.y (monmap changed)... 2024-07-15T06:59:43.540 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:43 smithi089 bash[79810]: audit 2024-07-15T06:59:42.190133+0000 mon.a (mon.0) 191 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-07-15T06:59:43.540 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:43 smithi089 bash[79810]: audit 2024-07-15T06:59:42.191816+0000 mon.a (mon.0) 192 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-07-15T06:59:43.540 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:43 smithi089 bash[79810]: audit 2024-07-15T06:59:42.193606+0000 mon.a (mon.0) 193 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:59:43.541 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:43 smithi089 bash[79810]: cephadm 2024-07-15T06:59:42.195191+0000 mgr.y (mgr.25244) 110 : cephadm [INF] Reconfiguring daemon mgr.y on smithi089 2024-07-15T06:59:43.541 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:43 smithi089 bash[79810]: cluster 2024-07-15T06:59:42.759596+0000 mgr.y (mgr.25244) 111 : cluster [DBG] pgmap v82: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:59:43.804 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:59:43 smithi134 bash[60729]: ts=2024-07-15T06:59:43.508Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi134\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi134\", job=\"node\", machine=\"x86_64\", nodename=\"smithi134\", release=\"5.15.0-113-generic\", sysname=\"Linux\", version=\"#123-Ubuntu SMP Mon Jun 10 08:16:17 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", domainname=\"(none)\", instance=\"smithi134\", job=\"node\", machine=\"x86_64\", nodename=\"smithi134\", release=\"5.15.0-113-generic\", sysname=\"Linux\", version=\"#123-Ubuntu SMP Mon Jun 10 08:16:17 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:59:44.619 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:59:44 smithi089 bash[72894]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:59:44] "GET /metrics HTTP/1.1" 200 33863 "" "Prometheus/2.43.0" 2024-07-15T06:59:44.804 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:59:44 smithi134 bash[60729]: ts=2024-07-15T06:59:44.454Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:59:44.804 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:59:44 smithi134 bash[60729]: ts=2024-07-15T06:59:44.455Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:59:45.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:45 smithi134 bash[67624]: audit 2024-07-15T06:59:44.400358+0000 mon.a (mon.0) 194 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:59:45.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:45 smithi134 bash[67624]: audit 2024-07-15T06:59:44.409209+0000 mon.a (mon.0) 195 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:59:45.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:45 smithi134 bash[67624]: cephadm 2024-07-15T06:59:44.410539+0000 mgr.y (mgr.25244) 112 : cephadm [INF] Reconfiguring mon.c (monmap changed)... 2024-07-15T06:59:45.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:45 smithi134 bash[67624]: audit 2024-07-15T06:59:44.410985+0000 mon.a (mon.0) 196 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-07-15T06:59:45.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:45 smithi134 bash[67624]: audit 2024-07-15T06:59:44.411957+0000 mon.a (mon.0) 197 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-07-15T06:59:45.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:45 smithi134 bash[67624]: audit 2024-07-15T06:59:44.412821+0000 mon.a (mon.0) 198 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:59:45.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:45 smithi134 bash[67624]: cephadm 2024-07-15T06:59:44.413883+0000 mgr.y (mgr.25244) 113 : cephadm [INF] Reconfiguring daemon mon.c on smithi089 2024-07-15T06:59:45.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:45 smithi134 bash[67624]: cluster 2024-07-15T06:59:44.760077+0000 mgr.y (mgr.25244) 114 : cluster [DBG] pgmap v83: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:59:45.839 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:45 smithi089 bash[79810]: audit 2024-07-15T06:59:44.400358+0000 mon.a (mon.0) 194 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:59:45.839 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:45 smithi089 bash[79810]: audit 2024-07-15T06:59:44.409209+0000 mon.a (mon.0) 195 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:59:45.839 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:45 smithi089 bash[79810]: cephadm 2024-07-15T06:59:44.410539+0000 mgr.y (mgr.25244) 112 : cephadm [INF] Reconfiguring mon.c (monmap changed)... 2024-07-15T06:59:45.839 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:45 smithi089 bash[79810]: audit 2024-07-15T06:59:44.410985+0000 mon.a (mon.0) 196 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-07-15T06:59:45.839 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:45 smithi089 bash[79810]: audit 2024-07-15T06:59:44.411957+0000 mon.a (mon.0) 197 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-07-15T06:59:45.839 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:45 smithi089 bash[79810]: audit 2024-07-15T06:59:44.412821+0000 mon.a (mon.0) 198 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:59:45.840 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:45 smithi089 bash[79810]: cephadm 2024-07-15T06:59:44.413883+0000 mgr.y (mgr.25244) 113 : cephadm [INF] Reconfiguring daemon mon.c on smithi089 2024-07-15T06:59:45.840 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:45 smithi089 bash[79810]: cluster 2024-07-15T06:59:44.760077+0000 mgr.y (mgr.25244) 114 : cluster [DBG] pgmap v83: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:59:45.840 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:45 smithi089 bash[82202]: audit 2024-07-15T06:59:44.400358+0000 mon.a (mon.0) 194 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:59:45.840 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:45 smithi089 bash[82202]: audit 2024-07-15T06:59:44.409209+0000 mon.a (mon.0) 195 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:59:45.840 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:45 smithi089 bash[82202]: cephadm 2024-07-15T06:59:44.410539+0000 mgr.y (mgr.25244) 112 : cephadm [INF] Reconfiguring mon.c (monmap changed)... 2024-07-15T06:59:45.840 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:45 smithi089 bash[82202]: audit 2024-07-15T06:59:44.410985+0000 mon.a (mon.0) 196 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-07-15T06:59:45.840 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:45 smithi089 bash[82202]: audit 2024-07-15T06:59:44.411957+0000 mon.a (mon.0) 197 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-07-15T06:59:45.840 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:45 smithi089 bash[82202]: audit 2024-07-15T06:59:44.412821+0000 mon.a (mon.0) 198 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:59:45.840 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:45 smithi089 bash[82202]: cephadm 2024-07-15T06:59:44.413883+0000 mgr.y (mgr.25244) 113 : cephadm [INF] Reconfiguring daemon mon.c on smithi089 2024-07-15T06:59:45.841 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:45 smithi089 bash[82202]: cluster 2024-07-15T06:59:44.760077+0000 mgr.y (mgr.25244) 114 : cluster [DBG] pgmap v83: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:59:47.972 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:47 smithi089 bash[82202]: audit 2024-07-15T06:59:46.709110+0000 mon.a (mon.0) 199 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:59:47.973 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:47 smithi089 bash[82202]: audit 2024-07-15T06:59:46.717186+0000 mon.a (mon.0) 200 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:59:47.973 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:47 smithi089 bash[82202]: cephadm 2024-07-15T06:59:46.718588+0000 mgr.y (mgr.25244) 115 : cephadm [INF] Reconfiguring osd.0 (monmap changed)... 2024-07-15T06:59:47.973 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:47 smithi089 bash[82202]: audit 2024-07-15T06:59:46.719099+0000 mon.a (mon.0) 201 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-07-15T06:59:47.973 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:47 smithi089 bash[82202]: audit 2024-07-15T06:59:46.720425+0000 mon.a (mon.0) 202 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:59:47.973 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:47 smithi089 bash[82202]: cephadm 2024-07-15T06:59:46.724651+0000 mgr.y (mgr.25244) 116 : cephadm [INF] Reconfiguring daemon osd.0 on smithi089 2024-07-15T06:59:47.973 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:47 smithi089 bash[82202]: cluster 2024-07-15T06:59:46.760759+0000 mgr.y (mgr.25244) 117 : cluster [DBG] pgmap v84: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:59:47.973 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:47 smithi089 bash[79810]: audit 2024-07-15T06:59:46.709110+0000 mon.a (mon.0) 199 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:59:47.973 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:47 smithi089 bash[79810]: audit 2024-07-15T06:59:46.717186+0000 mon.a (mon.0) 200 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:59:47.973 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:47 smithi089 bash[79810]: cephadm 2024-07-15T06:59:46.718588+0000 mgr.y (mgr.25244) 115 : cephadm [INF] Reconfiguring osd.0 (monmap changed)... 2024-07-15T06:59:47.974 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:47 smithi089 bash[79810]: audit 2024-07-15T06:59:46.719099+0000 mon.a (mon.0) 201 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-07-15T06:59:47.974 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:47 smithi089 bash[79810]: audit 2024-07-15T06:59:46.720425+0000 mon.a (mon.0) 202 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:59:47.974 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:47 smithi089 bash[79810]: cephadm 2024-07-15T06:59:46.724651+0000 mgr.y (mgr.25244) 116 : cephadm [INF] Reconfiguring daemon osd.0 on smithi089 2024-07-15T06:59:47.974 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:47 smithi089 bash[79810]: cluster 2024-07-15T06:59:46.760759+0000 mgr.y (mgr.25244) 117 : cluster [DBG] pgmap v84: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:59:48.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:47 smithi134 bash[67624]: audit 2024-07-15T06:59:46.709110+0000 mon.a (mon.0) 199 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:59:48.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:47 smithi134 bash[67624]: audit 2024-07-15T06:59:46.717186+0000 mon.a (mon.0) 200 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:59:48.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:47 smithi134 bash[67624]: cephadm 2024-07-15T06:59:46.718588+0000 mgr.y (mgr.25244) 115 : cephadm [INF] Reconfiguring osd.0 (monmap changed)... 2024-07-15T06:59:48.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:47 smithi134 bash[67624]: audit 2024-07-15T06:59:46.719099+0000 mon.a (mon.0) 201 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-07-15T06:59:48.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:47 smithi134 bash[67624]: audit 2024-07-15T06:59:46.720425+0000 mon.a (mon.0) 202 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:59:48.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:47 smithi134 bash[67624]: cephadm 2024-07-15T06:59:46.724651+0000 mgr.y (mgr.25244) 116 : cephadm [INF] Reconfiguring daemon osd.0 on smithi089 2024-07-15T06:59:48.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:47 smithi134 bash[67624]: cluster 2024-07-15T06:59:46.760759+0000 mgr.y (mgr.25244) 117 : cluster [DBG] pgmap v84: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:59:49.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:48 smithi134 bash[67624]: cluster 2024-07-15T06:59:48.761352+0000 mgr.y (mgr.25244) 118 : cluster [DBG] pgmap v85: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:59:49.055 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:59:48 smithi134 bash[60729]: ts=2024-07-15T06:59:48.662Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1102-g9a202024 (9a2020246f85463d4fdb115c94aa40b463caa1d2) quincy (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1102-g9a202024 (9a2020246f85463d4fdb115c94aa40b463caa1d2) quincy (stable)\", cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:59:49.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:48 smithi089 bash[79810]: cluster 2024-07-15T06:59:48.761352+0000 mgr.y (mgr.25244) 118 : cluster [DBG] pgmap v85: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:59:49.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:48 smithi089 bash[82202]: cluster 2024-07-15T06:59:48.761352+0000 mgr.y (mgr.25244) 118 : cluster [DBG] pgmap v85: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:59:50.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:49 smithi134 bash[67624]: audit 2024-07-15T06:59:48.934194+0000 mon.a (mon.0) 203 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:59:50.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:49 smithi134 bash[67624]: audit 2024-07-15T06:59:48.942343+0000 mon.a (mon.0) 204 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:59:50.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:49 smithi134 bash[67624]: cephadm 2024-07-15T06:59:48.943773+0000 mgr.y (mgr.25244) 119 : cephadm [INF] Reconfiguring osd.1 (monmap changed)... 2024-07-15T06:59:50.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:49 smithi134 bash[67624]: audit 2024-07-15T06:59:48.944114+0000 mon.a (mon.0) 205 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-07-15T06:59:50.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:49 smithi134 bash[67624]: audit 2024-07-15T06:59:48.945042+0000 mon.a (mon.0) 206 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:59:50.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:49 smithi134 bash[67624]: cephadm 2024-07-15T06:59:48.947442+0000 mgr.y (mgr.25244) 120 : cephadm [INF] Reconfiguring daemon osd.1 on smithi089 2024-07-15T06:59:50.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:49 smithi134 bash[67624]: audit 2024-07-15T06:59:49.787186+0000 mon.a (mon.0) 207 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:59:50.346 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:49 smithi089 bash[79810]: audit 2024-07-15T06:59:48.934194+0000 mon.a (mon.0) 203 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:59:50.347 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:49 smithi089 bash[79810]: audit 2024-07-15T06:59:48.942343+0000 mon.a (mon.0) 204 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:59:50.347 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:49 smithi089 bash[79810]: cephadm 2024-07-15T06:59:48.943773+0000 mgr.y (mgr.25244) 119 : cephadm [INF] Reconfiguring osd.1 (monmap changed)... 2024-07-15T06:59:50.347 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:49 smithi089 bash[79810]: audit 2024-07-15T06:59:48.944114+0000 mon.a (mon.0) 205 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-07-15T06:59:50.347 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:49 smithi089 bash[79810]: audit 2024-07-15T06:59:48.945042+0000 mon.a (mon.0) 206 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:59:50.347 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:49 smithi089 bash[79810]: cephadm 2024-07-15T06:59:48.947442+0000 mgr.y (mgr.25244) 120 : cephadm [INF] Reconfiguring daemon osd.1 on smithi089 2024-07-15T06:59:50.347 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:49 smithi089 bash[79810]: audit 2024-07-15T06:59:49.787186+0000 mon.a (mon.0) 207 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:59:50.347 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:49 smithi089 bash[82202]: audit 2024-07-15T06:59:48.934194+0000 mon.a (mon.0) 203 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:59:50.347 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:49 smithi089 bash[82202]: audit 2024-07-15T06:59:48.942343+0000 mon.a (mon.0) 204 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:59:50.347 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:49 smithi089 bash[82202]: cephadm 2024-07-15T06:59:48.943773+0000 mgr.y (mgr.25244) 119 : cephadm [INF] Reconfiguring osd.1 (monmap changed)... 2024-07-15T06:59:50.348 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:49 smithi089 bash[82202]: audit 2024-07-15T06:59:48.944114+0000 mon.a (mon.0) 205 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-07-15T06:59:50.348 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:49 smithi089 bash[82202]: audit 2024-07-15T06:59:48.945042+0000 mon.a (mon.0) 206 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:59:50.348 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:49 smithi089 bash[82202]: cephadm 2024-07-15T06:59:48.947442+0000 mgr.y (mgr.25244) 120 : cephadm [INF] Reconfiguring daemon osd.1 on smithi089 2024-07-15T06:59:50.348 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:49 smithi089 bash[82202]: audit 2024-07-15T06:59:49.787186+0000 mon.a (mon.0) 207 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T06:59:51.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:50 smithi089 bash[79810]: cluster 2024-07-15T06:59:50.762337+0000 mgr.y (mgr.25244) 121 : cluster [DBG] pgmap v86: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:59:51.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:50 smithi089 bash[82202]: cluster 2024-07-15T06:59:50.762337+0000 mgr.y (mgr.25244) 121 : cluster [DBG] pgmap v86: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:59:51.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:50 smithi134 bash[67624]: cluster 2024-07-15T06:59:50.762337+0000 mgr.y (mgr.25244) 121 : cluster [DBG] pgmap v86: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:59:52.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:52 smithi089 bash[79810]: audit 2024-07-15T06:59:51.328057+0000 mon.a (mon.0) 208 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:59:52.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:52 smithi089 bash[79810]: audit 2024-07-15T06:59:51.335951+0000 mon.a (mon.0) 209 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:59:52.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:52 smithi089 bash[79810]: cephadm 2024-07-15T06:59:51.337152+0000 mgr.y (mgr.25244) 122 : cephadm [INF] Reconfiguring osd.2 (monmap changed)... 2024-07-15T06:59:52.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:52 smithi089 bash[79810]: audit 2024-07-15T06:59:51.337586+0000 mon.a (mon.0) 210 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-07-15T06:59:52.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:52 smithi089 bash[79810]: audit 2024-07-15T06:59:51.338849+0000 mon.a (mon.0) 211 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:59:52.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:52 smithi089 bash[79810]: cephadm 2024-07-15T06:59:51.342564+0000 mgr.y (mgr.25244) 123 : cephadm [INF] Reconfiguring daemon osd.2 on smithi089 2024-07-15T06:59:52.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:52 smithi089 bash[82202]: audit 2024-07-15T06:59:51.328057+0000 mon.a (mon.0) 208 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:59:52.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:52 smithi089 bash[82202]: audit 2024-07-15T06:59:51.335951+0000 mon.a (mon.0) 209 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:59:52.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:52 smithi089 bash[82202]: cephadm 2024-07-15T06:59:51.337152+0000 mgr.y (mgr.25244) 122 : cephadm [INF] Reconfiguring osd.2 (monmap changed)... 2024-07-15T06:59:52.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:52 smithi089 bash[82202]: audit 2024-07-15T06:59:51.337586+0000 mon.a (mon.0) 210 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-07-15T06:59:52.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:52 smithi089 bash[82202]: audit 2024-07-15T06:59:51.338849+0000 mon.a (mon.0) 211 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:59:52.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:52 smithi089 bash[82202]: cephadm 2024-07-15T06:59:51.342564+0000 mgr.y (mgr.25244) 123 : cephadm [INF] Reconfiguring daemon osd.2 on smithi089 2024-07-15T06:59:52.804 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:59:52 smithi134 bash[60729]: ts=2024-07-15T06:59:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:59:52.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:52 smithi134 bash[67624]: audit 2024-07-15T06:59:51.328057+0000 mon.a (mon.0) 208 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:59:52.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:52 smithi134 bash[67624]: audit 2024-07-15T06:59:51.335951+0000 mon.a (mon.0) 209 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:59:52.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:52 smithi134 bash[67624]: cephadm 2024-07-15T06:59:51.337152+0000 mgr.y (mgr.25244) 122 : cephadm [INF] Reconfiguring osd.2 (monmap changed)... 2024-07-15T06:59:52.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:52 smithi134 bash[67624]: audit 2024-07-15T06:59:51.337586+0000 mon.a (mon.0) 210 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-07-15T06:59:52.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:52 smithi134 bash[67624]: audit 2024-07-15T06:59:51.338849+0000 mon.a (mon.0) 211 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:59:52.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:52 smithi134 bash[67624]: cephadm 2024-07-15T06:59:51.342564+0000 mgr.y (mgr.25244) 123 : cephadm [INF] Reconfiguring daemon osd.2 on smithi089 2024-07-15T06:59:53.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:53 smithi089 bash[79810]: cluster 2024-07-15T06:59:52.762964+0000 mgr.y (mgr.25244) 124 : cluster [DBG] pgmap v87: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:59:53.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:53 smithi089 bash[82202]: cluster 2024-07-15T06:59:52.762964+0000 mgr.y (mgr.25244) 124 : cluster [DBG] pgmap v87: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:59:53.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:53 smithi134 bash[67624]: cluster 2024-07-15T06:59:52.762964+0000 mgr.y (mgr.25244) 124 : cluster [DBG] pgmap v87: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:59:53.805 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:59:53 smithi134 bash[60729]: ts=2024-07-15T06:59:53.508Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi134\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi134\", job=\"node\", machine=\"x86_64\", nodename=\"smithi134\", release=\"5.15.0-113-generic\", sysname=\"Linux\", version=\"#123-Ubuntu SMP Mon Jun 10 08:16:17 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", domainname=\"(none)\", instance=\"smithi134\", job=\"node\", machine=\"x86_64\", nodename=\"smithi134\", release=\"5.15.0-113-generic\", sysname=\"Linux\", version=\"#123-Ubuntu SMP Mon Jun 10 08:16:17 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:59:54.369 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 06:59:54 smithi089 bash[72894]: ::ffff:172.21.15.134 - - [15/Jul/2024:06:59:54] "GET /metrics HTTP/1.1" 200 33859 "" "Prometheus/2.43.0" 2024-07-15T06:59:54.703 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:59:54 smithi134 bash[60729]: ts=2024-07-15T06:59:54.453Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:59:54.703 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:59:54 smithi134 bash[60729]: ts=2024-07-15T06:59:54.454Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:59:55.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:54 smithi134 bash[67624]: audit 2024-07-15T06:59:53.709880+0000 mon.a (mon.0) 212 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:59:55.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:54 smithi134 bash[67624]: audit 2024-07-15T06:59:53.720519+0000 mon.a (mon.0) 213 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:59:55.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:54 smithi134 bash[67624]: cephadm 2024-07-15T06:59:53.722187+0000 mgr.y (mgr.25244) 125 : cephadm [INF] Reconfiguring osd.3 (monmap changed)... 2024-07-15T06:59:55.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:54 smithi134 bash[67624]: audit 2024-07-15T06:59:53.722601+0000 mon.a (mon.0) 214 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-07-15T06:59:55.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:54 smithi134 bash[67624]: audit 2024-07-15T06:59:53.723844+0000 mon.a (mon.0) 215 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:59:55.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:54 smithi134 bash[67624]: cephadm 2024-07-15T06:59:53.727863+0000 mgr.y (mgr.25244) 126 : cephadm [INF] Reconfiguring daemon osd.3 on smithi089 2024-07-15T06:59:55.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:54 smithi089 bash[79810]: audit 2024-07-15T06:59:53.709880+0000 mon.a (mon.0) 212 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:59:55.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:54 smithi089 bash[79810]: audit 2024-07-15T06:59:53.720519+0000 mon.a (mon.0) 213 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:59:55.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:54 smithi089 bash[79810]: cephadm 2024-07-15T06:59:53.722187+0000 mgr.y (mgr.25244) 125 : cephadm [INF] Reconfiguring osd.3 (monmap changed)... 2024-07-15T06:59:55.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:54 smithi089 bash[79810]: audit 2024-07-15T06:59:53.722601+0000 mon.a (mon.0) 214 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-07-15T06:59:55.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:54 smithi089 bash[79810]: audit 2024-07-15T06:59:53.723844+0000 mon.a (mon.0) 215 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:59:55.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:54 smithi089 bash[79810]: cephadm 2024-07-15T06:59:53.727863+0000 mgr.y (mgr.25244) 126 : cephadm [INF] Reconfiguring daemon osd.3 on smithi089 2024-07-15T06:59:55.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:54 smithi089 bash[82202]: audit 2024-07-15T06:59:53.709880+0000 mon.a (mon.0) 212 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:59:55.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:54 smithi089 bash[82202]: audit 2024-07-15T06:59:53.720519+0000 mon.a (mon.0) 213 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:59:55.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:54 smithi089 bash[82202]: cephadm 2024-07-15T06:59:53.722187+0000 mgr.y (mgr.25244) 125 : cephadm [INF] Reconfiguring osd.3 (monmap changed)... 2024-07-15T06:59:55.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:54 smithi089 bash[82202]: audit 2024-07-15T06:59:53.722601+0000 mon.a (mon.0) 214 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-07-15T06:59:55.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:54 smithi089 bash[82202]: audit 2024-07-15T06:59:53.723844+0000 mon.a (mon.0) 215 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:59:55.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:54 smithi089 bash[82202]: cephadm 2024-07-15T06:59:53.727863+0000 mgr.y (mgr.25244) 126 : cephadm [INF] Reconfiguring daemon osd.3 on smithi089 2024-07-15T06:59:56.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:55 smithi134 bash[67624]: cluster 2024-07-15T06:59:54.763713+0000 mgr.y (mgr.25244) 127 : cluster [DBG] pgmap v88: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:59:56.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:55 smithi089 bash[79810]: cluster 2024-07-15T06:59:54.763713+0000 mgr.y (mgr.25244) 127 : cluster [DBG] pgmap v88: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:59:56.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:55 smithi089 bash[82202]: cluster 2024-07-15T06:59:54.763713+0000 mgr.y (mgr.25244) 127 : cluster [DBG] pgmap v88: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:59:57.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:57 smithi089 bash[79810]: audit 2024-07-15T06:59:56.076422+0000 mon.a (mon.0) 216 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:59:57.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:57 smithi089 bash[79810]: audit 2024-07-15T06:59:56.083380+0000 mon.a (mon.0) 217 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:59:57.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:57 smithi089 bash[79810]: cephadm 2024-07-15T06:59:56.086134+0000 mgr.y (mgr.25244) 128 : cephadm [INF] Reconfiguring mds.foo.smithi089.cczuoz (monmap changed)... 2024-07-15T06:59:57.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:57 smithi089 bash[79810]: audit 2024-07-15T06:59:56.086824+0000 mon.a (mon.0) 218 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi089.cczuoz", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-07-15T06:59:57.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:57 smithi089 bash[79810]: audit 2024-07-15T06:59:56.088750+0000 mon.a (mon.0) 219 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:59:57.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:57 smithi089 bash[79810]: cephadm 2024-07-15T06:59:56.090560+0000 mgr.y (mgr.25244) 129 : cephadm [INF] Reconfiguring daemon mds.foo.smithi089.cczuoz on smithi089 2024-07-15T06:59:57.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:57 smithi089 bash[79810]: cluster 2024-07-15T06:59:56.764783+0000 mgr.y (mgr.25244) 130 : cluster [DBG] pgmap v89: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:59:57.371 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:57 smithi089 bash[82202]: audit 2024-07-15T06:59:56.076422+0000 mon.a (mon.0) 216 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:59:57.371 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:57 smithi089 bash[82202]: audit 2024-07-15T06:59:56.083380+0000 mon.a (mon.0) 217 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:59:57.371 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:57 smithi089 bash[82202]: cephadm 2024-07-15T06:59:56.086134+0000 mgr.y (mgr.25244) 128 : cephadm [INF] Reconfiguring mds.foo.smithi089.cczuoz (monmap changed)... 2024-07-15T06:59:57.371 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:57 smithi089 bash[82202]: audit 2024-07-15T06:59:56.086824+0000 mon.a (mon.0) 218 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi089.cczuoz", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-07-15T06:59:57.371 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:57 smithi089 bash[82202]: audit 2024-07-15T06:59:56.088750+0000 mon.a (mon.0) 219 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:59:57.371 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:57 smithi089 bash[82202]: cephadm 2024-07-15T06:59:56.090560+0000 mgr.y (mgr.25244) 129 : cephadm [INF] Reconfiguring daemon mds.foo.smithi089.cczuoz on smithi089 2024-07-15T06:59:57.371 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:57 smithi089 bash[82202]: cluster 2024-07-15T06:59:56.764783+0000 mgr.y (mgr.25244) 130 : cluster [DBG] pgmap v89: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:59:57.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:57 smithi134 bash[67624]: audit 2024-07-15T06:59:56.076422+0000 mon.a (mon.0) 216 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:59:57.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:57 smithi134 bash[67624]: audit 2024-07-15T06:59:56.083380+0000 mon.a (mon.0) 217 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:59:57.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:57 smithi134 bash[67624]: cephadm 2024-07-15T06:59:56.086134+0000 mgr.y (mgr.25244) 128 : cephadm [INF] Reconfiguring mds.foo.smithi089.cczuoz (monmap changed)... 2024-07-15T06:59:57.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:57 smithi134 bash[67624]: audit 2024-07-15T06:59:56.086824+0000 mon.a (mon.0) 218 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi089.cczuoz", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-07-15T06:59:57.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:57 smithi134 bash[67624]: audit 2024-07-15T06:59:56.088750+0000 mon.a (mon.0) 219 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:59:57.557 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:57 smithi134 bash[67624]: cephadm 2024-07-15T06:59:56.090560+0000 mgr.y (mgr.25244) 129 : cephadm [INF] Reconfiguring daemon mds.foo.smithi089.cczuoz on smithi089 2024-07-15T06:59:57.557 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:57 smithi134 bash[67624]: cluster 2024-07-15T06:59:56.764783+0000 mgr.y (mgr.25244) 130 : cluster [DBG] pgmap v89: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:59:58.919 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 06:59:58 smithi134 bash[60729]: ts=2024-07-15T06:59:58.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1102-g9a202024 (9a2020246f85463d4fdb115c94aa40b463caa1d2) quincy (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1102-g9a202024 (9a2020246f85463d4fdb115c94aa40b463caa1d2) quincy (stable)\", cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T06:59:59.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:59 smithi134 bash[67624]: audit 2024-07-15T06:59:58.451451+0000 mon.a (mon.0) 220 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:59:59.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:59 smithi134 bash[67624]: audit 2024-07-15T06:59:58.459342+0000 mon.a (mon.0) 221 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:59:59.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:59 smithi134 bash[67624]: cephadm 2024-07-15T06:59:58.460682+0000 mgr.y (mgr.25244) 131 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2024-07-15T06:59:59.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:59 smithi134 bash[67624]: audit 2024-07-15T06:59:58.461110+0000 mon.a (mon.0) 222 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-07-15T06:59:59.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:59 smithi134 bash[67624]: audit 2024-07-15T06:59:58.462147+0000 mon.a (mon.0) 223 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-07-15T06:59:59.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:59 smithi134 bash[67624]: audit 2024-07-15T06:59:58.463039+0000 mon.a (mon.0) 224 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:59:59.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:59 smithi134 bash[67624]: cephadm 2024-07-15T06:59:58.464332+0000 mgr.y (mgr.25244) 132 : cephadm [INF] Reconfiguring daemon mon.b on smithi134 2024-07-15T06:59:59.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 06:59:59 smithi134 bash[67624]: cluster 2024-07-15T06:59:58.765402+0000 mgr.y (mgr.25244) 133 : cluster [DBG] pgmap v90: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:59:59.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:59 smithi089 bash[79810]: audit 2024-07-15T06:59:58.451451+0000 mon.a (mon.0) 220 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:59:59.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:59 smithi089 bash[79810]: audit 2024-07-15T06:59:58.459342+0000 mon.a (mon.0) 221 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:59:59.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:59 smithi089 bash[79810]: cephadm 2024-07-15T06:59:58.460682+0000 mgr.y (mgr.25244) 131 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2024-07-15T06:59:59.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:59 smithi089 bash[79810]: audit 2024-07-15T06:59:58.461110+0000 mon.a (mon.0) 222 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-07-15T06:59:59.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:59 smithi089 bash[79810]: audit 2024-07-15T06:59:58.462147+0000 mon.a (mon.0) 223 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-07-15T06:59:59.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:59 smithi089 bash[79810]: audit 2024-07-15T06:59:58.463039+0000 mon.a (mon.0) 224 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:59:59.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:59 smithi089 bash[79810]: cephadm 2024-07-15T06:59:58.464332+0000 mgr.y (mgr.25244) 132 : cephadm [INF] Reconfiguring daemon mon.b on smithi134 2024-07-15T06:59:59.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 06:59:59 smithi089 bash[79810]: cluster 2024-07-15T06:59:58.765402+0000 mgr.y (mgr.25244) 133 : cluster [DBG] pgmap v90: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T06:59:59.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:59 smithi089 bash[82202]: audit 2024-07-15T06:59:58.451451+0000 mon.a (mon.0) 220 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:59:59.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:59 smithi089 bash[82202]: audit 2024-07-15T06:59:58.459342+0000 mon.a (mon.0) 221 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T06:59:59.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:59 smithi089 bash[82202]: cephadm 2024-07-15T06:59:58.460682+0000 mgr.y (mgr.25244) 131 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2024-07-15T06:59:59.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:59 smithi089 bash[82202]: audit 2024-07-15T06:59:58.461110+0000 mon.a (mon.0) 222 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-07-15T06:59:59.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:59 smithi089 bash[82202]: audit 2024-07-15T06:59:58.462147+0000 mon.a (mon.0) 223 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-07-15T06:59:59.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:59 smithi089 bash[82202]: audit 2024-07-15T06:59:58.463039+0000 mon.a (mon.0) 224 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T06:59:59.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:59 smithi089 bash[82202]: cephadm 2024-07-15T06:59:58.464332+0000 mgr.y (mgr.25244) 132 : cephadm [INF] Reconfiguring daemon mon.b on smithi134 2024-07-15T06:59:59.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 06:59:59 smithi089 bash[82202]: cluster 2024-07-15T06:59:58.765402+0000 mgr.y (mgr.25244) 133 : cluster [DBG] pgmap v90: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:00:00.369 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:00 smithi089 bash[79810]: debug 2024-07-15T06:59:59.995+0000 7fb563a0a640 -1 log_channel(cluster) log [ERR] : overall HEALTH_ERR 8 osds(s) are not reachable 2024-07-15T07:00:00.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:00 smithi134 bash[67624]: cluster 2024-07-15T07:00:00.000139+0000 mon.a (mon.0) 225 : cluster [ERR] overall HEALTH_ERR 8 osds(s) are not reachable 2024-07-15T07:00:00.869 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:00 smithi089 bash[79810]: cluster 2024-07-15T07:00:00.000139+0000 mon.a (mon.0) 225 : cluster [ERR] overall HEALTH_ERR 8 osds(s) are not reachable 2024-07-15T07:00:00.870 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:00 smithi089 bash[82202]: cluster 2024-07-15T07:00:00.000139+0000 mon.a (mon.0) 225 : cluster [ERR] overall HEALTH_ERR 8 osds(s) are not reachable 2024-07-15T07:00:02.085 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:01 smithi134 bash[67624]: cluster 2024-07-15T07:00:00.766428+0000 mgr.y (mgr.25244) 134 : cluster [DBG] pgmap v91: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:00:02.085 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:01 smithi134 bash[67624]: audit 2024-07-15T07:00:00.812234+0000 mon.a (mon.0) 226 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:00:02.085 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:01 smithi134 bash[67624]: audit 2024-07-15T07:00:00.822686+0000 mon.a (mon.0) 227 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:00:02.085 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:01 smithi134 bash[67624]: audit 2024-07-15T07:00:00.825208+0000 mon.a (mon.0) 228 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-07-15T07:00:02.086 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:01 smithi134 bash[67624]: audit 2024-07-15T07:00:00.826944+0000 mon.a (mon.0) 229 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-07-15T07:00:02.086 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:01 smithi134 bash[67624]: audit 2024-07-15T07:00:00.828466+0000 mon.a (mon.0) 230 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T07:00:02.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:01 smithi089 bash[79810]: cluster 2024-07-15T07:00:00.766428+0000 mgr.y (mgr.25244) 134 : cluster [DBG] pgmap v91: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:00:02.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:01 smithi089 bash[79810]: audit 2024-07-15T07:00:00.812234+0000 mon.a (mon.0) 226 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:00:02.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:01 smithi089 bash[79810]: audit 2024-07-15T07:00:00.822686+0000 mon.a (mon.0) 227 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:00:02.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:01 smithi089 bash[79810]: audit 2024-07-15T07:00:00.825208+0000 mon.a (mon.0) 228 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-07-15T07:00:02.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:01 smithi089 bash[79810]: audit 2024-07-15T07:00:00.826944+0000 mon.a (mon.0) 229 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-07-15T07:00:02.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:01 smithi089 bash[79810]: audit 2024-07-15T07:00:00.828466+0000 mon.a (mon.0) 230 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T07:00:02.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:01 smithi089 bash[82202]: cluster 2024-07-15T07:00:00.766428+0000 mgr.y (mgr.25244) 134 : cluster [DBG] pgmap v91: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:00:02.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:01 smithi089 bash[82202]: audit 2024-07-15T07:00:00.812234+0000 mon.a (mon.0) 226 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:00:02.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:01 smithi089 bash[82202]: audit 2024-07-15T07:00:00.822686+0000 mon.a (mon.0) 227 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:00:02.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:01 smithi089 bash[82202]: audit 2024-07-15T07:00:00.825208+0000 mon.a (mon.0) 228 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-07-15T07:00:02.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:01 smithi089 bash[82202]: audit 2024-07-15T07:00:00.826944+0000 mon.a (mon.0) 229 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-07-15T07:00:02.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:01 smithi089 bash[82202]: audit 2024-07-15T07:00:00.828466+0000 mon.a (mon.0) 230 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T07:00:02.450 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:00:02 smithi134 bash[60729]: ts=2024-07-15T07:00:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T07:00:03.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:02 smithi089 bash[79810]: cephadm 2024-07-15T07:00:00.824616+0000 mgr.y (mgr.25244) 135 : cephadm [INF] Reconfiguring mgr.x (monmap changed)... 2024-07-15T07:00:03.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:02 smithi089 bash[79810]: cephadm 2024-07-15T07:00:00.830173+0000 mgr.y (mgr.25244) 136 : cephadm [INF] Reconfiguring daemon mgr.x on smithi134 2024-07-15T07:00:03.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:02 smithi089 bash[82202]: cephadm 2024-07-15T07:00:00.824616+0000 mgr.y (mgr.25244) 135 : cephadm [INF] Reconfiguring mgr.x (monmap changed)... 2024-07-15T07:00:03.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:02 smithi089 bash[82202]: cephadm 2024-07-15T07:00:00.830173+0000 mgr.y (mgr.25244) 136 : cephadm [INF] Reconfiguring daemon mgr.x on smithi134 2024-07-15T07:00:03.307 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:02 smithi134 bash[67624]: cephadm 2024-07-15T07:00:00.824616+0000 mgr.y (mgr.25244) 135 : cephadm [INF] Reconfiguring mgr.x (monmap changed)... 2024-07-15T07:00:03.308 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:02 smithi134 bash[67624]: cephadm 2024-07-15T07:00:00.830173+0000 mgr.y (mgr.25244) 136 : cephadm [INF] Reconfiguring daemon mgr.x on smithi134 2024-07-15T07:00:03.804 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:00:03 smithi134 bash[60729]: ts=2024-07-15T07:00:03.508Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi134\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi134\", job=\"node\", machine=\"x86_64\", nodename=\"smithi134\", release=\"5.15.0-113-generic\", sysname=\"Linux\", version=\"#123-Ubuntu SMP Mon Jun 10 08:16:17 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", domainname=\"(none)\", instance=\"smithi134\", job=\"node\", machine=\"x86_64\", nodename=\"smithi134\", release=\"5.15.0-113-generic\", sysname=\"Linux\", version=\"#123-Ubuntu SMP Mon Jun 10 08:16:17 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T07:00:04.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:03 smithi089 bash[79810]: cluster 2024-07-15T07:00:02.767009+0000 mgr.y (mgr.25244) 137 : cluster [DBG] pgmap v92: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:00:04.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:03 smithi089 bash[79810]: audit 2024-07-15T07:00:02.905665+0000 mon.a (mon.0) 231 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:00:04.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:03 smithi089 bash[79810]: audit 2024-07-15T07:00:02.916009+0000 mon.a (mon.0) 232 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:00:04.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:03 smithi089 bash[79810]: cephadm 2024-07-15T07:00:02.917790+0000 mgr.y (mgr.25244) 138 : cephadm [INF] Reconfiguring osd.4 (monmap changed)... 2024-07-15T07:00:04.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:03 smithi089 bash[79810]: audit 2024-07-15T07:00:02.918364+0000 mon.a (mon.0) 233 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-07-15T07:00:04.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:03 smithi089 bash[79810]: audit 2024-07-15T07:00:02.920065+0000 mon.a (mon.0) 234 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T07:00:04.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:03 smithi089 bash[79810]: cephadm 2024-07-15T07:00:02.924391+0000 mgr.y (mgr.25244) 139 : cephadm [INF] Reconfiguring daemon osd.4 on smithi134 2024-07-15T07:00:04.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:03 smithi089 bash[82202]: cluster 2024-07-15T07:00:02.767009+0000 mgr.y (mgr.25244) 137 : cluster [DBG] pgmap v92: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:00:04.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:03 smithi089 bash[82202]: audit 2024-07-15T07:00:02.905665+0000 mon.a (mon.0) 231 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:00:04.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:03 smithi089 bash[82202]: audit 2024-07-15T07:00:02.916009+0000 mon.a (mon.0) 232 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:00:04.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:03 smithi089 bash[82202]: cephadm 2024-07-15T07:00:02.917790+0000 mgr.y (mgr.25244) 138 : cephadm [INF] Reconfiguring osd.4 (monmap changed)... 2024-07-15T07:00:04.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:03 smithi089 bash[82202]: audit 2024-07-15T07:00:02.918364+0000 mon.a (mon.0) 233 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-07-15T07:00:04.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:03 smithi089 bash[82202]: audit 2024-07-15T07:00:02.920065+0000 mon.a (mon.0) 234 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T07:00:04.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:03 smithi089 bash[82202]: cephadm 2024-07-15T07:00:02.924391+0000 mgr.y (mgr.25244) 139 : cephadm [INF] Reconfiguring daemon osd.4 on smithi134 2024-07-15T07:00:04.217 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:03 smithi134 bash[67624]: cluster 2024-07-15T07:00:02.767009+0000 mgr.y (mgr.25244) 137 : cluster [DBG] pgmap v92: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:00:04.217 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:03 smithi134 bash[67624]: audit 2024-07-15T07:00:02.905665+0000 mon.a (mon.0) 231 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:00:04.217 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:03 smithi134 bash[67624]: audit 2024-07-15T07:00:02.916009+0000 mon.a (mon.0) 232 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:00:04.217 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:03 smithi134 bash[67624]: cephadm 2024-07-15T07:00:02.917790+0000 mgr.y (mgr.25244) 138 : cephadm [INF] Reconfiguring osd.4 (monmap changed)... 2024-07-15T07:00:04.217 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:03 smithi134 bash[67624]: audit 2024-07-15T07:00:02.918364+0000 mon.a (mon.0) 233 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-07-15T07:00:04.217 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:03 smithi134 bash[67624]: audit 2024-07-15T07:00:02.920065+0000 mon.a (mon.0) 234 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T07:00:04.217 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:03 smithi134 bash[67624]: cephadm 2024-07-15T07:00:02.924391+0000 mgr.y (mgr.25244) 139 : cephadm [INF] Reconfiguring daemon osd.4 on smithi134 2024-07-15T07:00:04.554 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:00:04 smithi134 bash[60729]: ts=2024-07-15T07:00:04.454Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T07:00:04.555 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:00:04 smithi134 bash[60729]: ts=2024-07-15T07:00:04.455Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T07:00:04.619 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 07:00:04 smithi089 bash[72894]: ::ffff:172.21.15.134 - - [15/Jul/2024:07:00:04] "GET /metrics HTTP/1.1" 200 33859 "" "Prometheus/2.43.0" 2024-07-15T07:00:05.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:04 smithi089 bash[82202]: cluster 2024-07-15T07:00:04.767639+0000 mgr.y (mgr.25244) 140 : cluster [DBG] pgmap v93: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:00:05.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:04 smithi089 bash[82202]: audit 2024-07-15T07:00:04.787574+0000 mon.a (mon.0) 235 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:00:05.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:04 smithi089 bash[79810]: cluster 2024-07-15T07:00:04.767639+0000 mgr.y (mgr.25244) 140 : cluster [DBG] pgmap v93: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:00:05.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:04 smithi089 bash[79810]: audit 2024-07-15T07:00:04.787574+0000 mon.a (mon.0) 235 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:00:05.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:04 smithi134 bash[67624]: cluster 2024-07-15T07:00:04.767639+0000 mgr.y (mgr.25244) 140 : cluster [DBG] pgmap v93: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:00:05.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:04 smithi134 bash[67624]: audit 2024-07-15T07:00:04.787574+0000 mon.a (mon.0) 235 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:00:06.368 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:06 smithi134 bash[67624]: audit 2024-07-15T07:00:05.105767+0000 mon.a (mon.0) 236 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:00:06.368 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:06 smithi134 bash[67624]: audit 2024-07-15T07:00:05.116297+0000 mon.a (mon.0) 237 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:00:06.368 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:06 smithi134 bash[67624]: cephadm 2024-07-15T07:00:05.118275+0000 mgr.y (mgr.25244) 141 : cephadm [INF] Reconfiguring osd.5 (monmap changed)... 2024-07-15T07:00:06.368 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:06 smithi134 bash[67624]: audit 2024-07-15T07:00:05.118868+0000 mon.a (mon.0) 238 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-07-15T07:00:06.369 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:06 smithi134 bash[67624]: audit 2024-07-15T07:00:05.120544+0000 mon.a (mon.0) 239 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T07:00:06.369 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:06 smithi134 bash[67624]: cephadm 2024-07-15T07:00:05.124903+0000 mgr.y (mgr.25244) 142 : cephadm [INF] Reconfiguring daemon osd.5 on smithi134 2024-07-15T07:00:06.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:06 smithi089 bash[82202]: audit 2024-07-15T07:00:05.105767+0000 mon.a (mon.0) 236 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:00:06.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:06 smithi089 bash[82202]: audit 2024-07-15T07:00:05.116297+0000 mon.a (mon.0) 237 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:00:06.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:06 smithi089 bash[82202]: cephadm 2024-07-15T07:00:05.118275+0000 mgr.y (mgr.25244) 141 : cephadm [INF] Reconfiguring osd.5 (monmap changed)... 2024-07-15T07:00:06.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:06 smithi089 bash[82202]: audit 2024-07-15T07:00:05.118868+0000 mon.a (mon.0) 238 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-07-15T07:00:06.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:06 smithi089 bash[82202]: audit 2024-07-15T07:00:05.120544+0000 mon.a (mon.0) 239 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T07:00:06.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:06 smithi089 bash[82202]: cephadm 2024-07-15T07:00:05.124903+0000 mgr.y (mgr.25244) 142 : cephadm [INF] Reconfiguring daemon osd.5 on smithi134 2024-07-15T07:00:06.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:06 smithi089 bash[79810]: audit 2024-07-15T07:00:05.105767+0000 mon.a (mon.0) 236 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:00:06.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:06 smithi089 bash[79810]: audit 2024-07-15T07:00:05.116297+0000 mon.a (mon.0) 237 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:00:06.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:06 smithi089 bash[79810]: cephadm 2024-07-15T07:00:05.118275+0000 mgr.y (mgr.25244) 141 : cephadm [INF] Reconfiguring osd.5 (monmap changed)... 2024-07-15T07:00:06.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:06 smithi089 bash[79810]: audit 2024-07-15T07:00:05.118868+0000 mon.a (mon.0) 238 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-07-15T07:00:06.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:06 smithi089 bash[79810]: audit 2024-07-15T07:00:05.120544+0000 mon.a (mon.0) 239 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T07:00:06.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:06 smithi089 bash[79810]: cephadm 2024-07-15T07:00:05.124903+0000 mgr.y (mgr.25244) 142 : cephadm [INF] Reconfiguring daemon osd.5 on smithi134 2024-07-15T07:00:07.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:07 smithi089 bash[79810]: cluster 2024-07-15T07:00:06.768690+0000 mgr.y (mgr.25244) 143 : cluster [DBG] pgmap v94: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:00:07.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:07 smithi089 bash[82202]: cluster 2024-07-15T07:00:06.768690+0000 mgr.y (mgr.25244) 143 : cluster [DBG] pgmap v94: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:00:07.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:07 smithi134 bash[67624]: cluster 2024-07-15T07:00:06.768690+0000 mgr.y (mgr.25244) 143 : cluster [DBG] pgmap v94: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:00:08.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:08 smithi134 bash[67624]: audit 2024-07-15T07:00:07.229592+0000 mon.a (mon.0) 240 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:00:08.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:08 smithi134 bash[67624]: audit 2024-07-15T07:00:07.242019+0000 mon.a (mon.0) 241 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:00:08.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:08 smithi134 bash[67624]: cephadm 2024-07-15T07:00:07.243926+0000 mgr.y (mgr.25244) 144 : cephadm [INF] Reconfiguring osd.6 (monmap changed)... 2024-07-15T07:00:08.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:08 smithi134 bash[67624]: audit 2024-07-15T07:00:07.244533+0000 mon.a (mon.0) 242 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-07-15T07:00:08.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:08 smithi134 bash[67624]: audit 2024-07-15T07:00:07.246282+0000 mon.a (mon.0) 243 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T07:00:08.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:08 smithi134 bash[67624]: cephadm 2024-07-15T07:00:07.250692+0000 mgr.y (mgr.25244) 145 : cephadm [INF] Reconfiguring daemon osd.6 on smithi134 2024-07-15T07:00:08.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:08 smithi089 bash[79810]: audit 2024-07-15T07:00:07.229592+0000 mon.a (mon.0) 240 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:00:08.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:08 smithi089 bash[79810]: audit 2024-07-15T07:00:07.242019+0000 mon.a (mon.0) 241 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:00:08.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:08 smithi089 bash[79810]: cephadm 2024-07-15T07:00:07.243926+0000 mgr.y (mgr.25244) 144 : cephadm [INF] Reconfiguring osd.6 (monmap changed)... 2024-07-15T07:00:08.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:08 smithi089 bash[79810]: audit 2024-07-15T07:00:07.244533+0000 mon.a (mon.0) 242 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-07-15T07:00:08.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:08 smithi089 bash[79810]: audit 2024-07-15T07:00:07.246282+0000 mon.a (mon.0) 243 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T07:00:08.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:08 smithi089 bash[79810]: cephadm 2024-07-15T07:00:07.250692+0000 mgr.y (mgr.25244) 145 : cephadm [INF] Reconfiguring daemon osd.6 on smithi134 2024-07-15T07:00:08.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:08 smithi089 bash[82202]: audit 2024-07-15T07:00:07.229592+0000 mon.a (mon.0) 240 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:00:08.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:08 smithi089 bash[82202]: audit 2024-07-15T07:00:07.242019+0000 mon.a (mon.0) 241 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:00:08.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:08 smithi089 bash[82202]: cephadm 2024-07-15T07:00:07.243926+0000 mgr.y (mgr.25244) 144 : cephadm [INF] Reconfiguring osd.6 (monmap changed)... 2024-07-15T07:00:08.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:08 smithi089 bash[82202]: audit 2024-07-15T07:00:07.244533+0000 mon.a (mon.0) 242 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-07-15T07:00:08.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:08 smithi089 bash[82202]: audit 2024-07-15T07:00:07.246282+0000 mon.a (mon.0) 243 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T07:00:08.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:08 smithi089 bash[82202]: cephadm 2024-07-15T07:00:07.250692+0000 mgr.y (mgr.25244) 145 : cephadm [INF] Reconfiguring daemon osd.6 on smithi134 2024-07-15T07:00:08.917 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:00:08 smithi134 bash[60729]: ts=2024-07-15T07:00:08.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1102-g9a202024 (9a2020246f85463d4fdb115c94aa40b463caa1d2) quincy (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1102-g9a202024 (9a2020246f85463d4fdb115c94aa40b463caa1d2) quincy (stable)\", cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T07:00:09.561 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:09 smithi134 bash[67624]: cluster 2024-07-15T07:00:08.769354+0000 mgr.y (mgr.25244) 146 : cluster [DBG] pgmap v95: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:00:09.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:09 smithi089 bash[79810]: cluster 2024-07-15T07:00:08.769354+0000 mgr.y (mgr.25244) 146 : cluster [DBG] pgmap v95: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:00:09.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:09 smithi089 bash[82202]: cluster 2024-07-15T07:00:08.769354+0000 mgr.y (mgr.25244) 146 : cluster [DBG] pgmap v95: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:00:10.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:10 smithi134 bash[67624]: audit 2024-07-15T07:00:09.467377+0000 mon.a (mon.0) 244 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:00:10.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:10 smithi134 bash[67624]: audit 2024-07-15T07:00:09.477626+0000 mon.a (mon.0) 245 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:00:10.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:10 smithi134 bash[67624]: cephadm 2024-07-15T07:00:09.479683+0000 mgr.y (mgr.25244) 147 : cephadm [INF] Reconfiguring osd.7 (monmap changed)... 2024-07-15T07:00:10.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:10 smithi134 bash[67624]: audit 2024-07-15T07:00:09.480230+0000 mon.a (mon.0) 246 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-07-15T07:00:10.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:10 smithi134 bash[67624]: audit 2024-07-15T07:00:09.481870+0000 mon.a (mon.0) 247 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T07:00:10.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:10 smithi134 bash[67624]: cephadm 2024-07-15T07:00:09.486708+0000 mgr.y (mgr.25244) 148 : cephadm [INF] Reconfiguring daemon osd.7 on smithi134 2024-07-15T07:00:10.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:10 smithi089 bash[79810]: audit 2024-07-15T07:00:09.467377+0000 mon.a (mon.0) 244 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:00:10.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:10 smithi089 bash[79810]: audit 2024-07-15T07:00:09.477626+0000 mon.a (mon.0) 245 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:00:10.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:10 smithi089 bash[79810]: cephadm 2024-07-15T07:00:09.479683+0000 mgr.y (mgr.25244) 147 : cephadm [INF] Reconfiguring osd.7 (monmap changed)... 2024-07-15T07:00:10.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:10 smithi089 bash[79810]: audit 2024-07-15T07:00:09.480230+0000 mon.a (mon.0) 246 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-07-15T07:00:10.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:10 smithi089 bash[79810]: audit 2024-07-15T07:00:09.481870+0000 mon.a (mon.0) 247 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T07:00:10.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:10 smithi089 bash[79810]: cephadm 2024-07-15T07:00:09.486708+0000 mgr.y (mgr.25244) 148 : cephadm [INF] Reconfiguring daemon osd.7 on smithi134 2024-07-15T07:00:10.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:10 smithi089 bash[82202]: audit 2024-07-15T07:00:09.467377+0000 mon.a (mon.0) 244 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:00:10.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:10 smithi089 bash[82202]: audit 2024-07-15T07:00:09.477626+0000 mon.a (mon.0) 245 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:00:10.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:10 smithi089 bash[82202]: cephadm 2024-07-15T07:00:09.479683+0000 mgr.y (mgr.25244) 147 : cephadm [INF] Reconfiguring osd.7 (monmap changed)... 2024-07-15T07:00:10.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:10 smithi089 bash[82202]: audit 2024-07-15T07:00:09.480230+0000 mon.a (mon.0) 246 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-07-15T07:00:10.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:10 smithi089 bash[82202]: audit 2024-07-15T07:00:09.481870+0000 mon.a (mon.0) 247 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T07:00:10.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:10 smithi089 bash[82202]: cephadm 2024-07-15T07:00:09.486708+0000 mgr.y (mgr.25244) 148 : cephadm [INF] Reconfiguring daemon osd.7 on smithi134 2024-07-15T07:00:11.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:11 smithi134 bash[67624]: cluster 2024-07-15T07:00:10.770388+0000 mgr.y (mgr.25244) 149 : cluster [DBG] pgmap v96: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:00:11.869 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:11 smithi089 bash[79810]: cluster 2024-07-15T07:00:10.770388+0000 mgr.y (mgr.25244) 149 : cluster [DBG] pgmap v96: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:00:11.869 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:11 smithi089 bash[82202]: cluster 2024-07-15T07:00:10.770388+0000 mgr.y (mgr.25244) 149 : cluster [DBG] pgmap v96: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:00:12.635 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:00:12 smithi134 bash[60729]: ts=2024-07-15T07:00:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T07:00:12.999 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:12 smithi134 bash[67624]: audit 2024-07-15T07:00:11.641699+0000 mon.a (mon.0) 248 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:00:12.999 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:12 smithi134 bash[67624]: audit 2024-07-15T07:00:11.651495+0000 mon.a (mon.0) 249 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:00:12.999 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:12 smithi134 bash[67624]: cephadm 2024-07-15T07:00:11.654743+0000 mgr.y (mgr.25244) 150 : cephadm [INF] Reconfiguring mds.foo.smithi134.wugtzh (monmap changed)... 2024-07-15T07:00:13.000 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:12 smithi134 bash[67624]: audit 2024-07-15T07:00:11.655393+0000 mon.a (mon.0) 250 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi134.wugtzh", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-07-15T07:00:13.000 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:12 smithi134 bash[67624]: audit 2024-07-15T07:00:11.657355+0000 mon.a (mon.0) 251 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T07:00:13.000 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:12 smithi134 bash[67624]: cephadm 2024-07-15T07:00:11.659316+0000 mgr.y (mgr.25244) 151 : cephadm [INF] Reconfiguring daemon mds.foo.smithi134.wugtzh on smithi134 2024-07-15T07:00:13.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:12 smithi089 bash[79810]: audit 2024-07-15T07:00:11.641699+0000 mon.a (mon.0) 248 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:00:13.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:12 smithi089 bash[79810]: audit 2024-07-15T07:00:11.651495+0000 mon.a (mon.0) 249 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:00:13.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:12 smithi089 bash[79810]: cephadm 2024-07-15T07:00:11.654743+0000 mgr.y (mgr.25244) 150 : cephadm [INF] Reconfiguring mds.foo.smithi134.wugtzh (monmap changed)... 2024-07-15T07:00:13.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:12 smithi089 bash[79810]: audit 2024-07-15T07:00:11.655393+0000 mon.a (mon.0) 250 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi134.wugtzh", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-07-15T07:00:13.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:12 smithi089 bash[79810]: audit 2024-07-15T07:00:11.657355+0000 mon.a (mon.0) 251 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T07:00:13.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:12 smithi089 bash[79810]: cephadm 2024-07-15T07:00:11.659316+0000 mgr.y (mgr.25244) 151 : cephadm [INF] Reconfiguring daemon mds.foo.smithi134.wugtzh on smithi134 2024-07-15T07:00:13.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:12 smithi089 bash[82202]: audit 2024-07-15T07:00:11.641699+0000 mon.a (mon.0) 248 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:00:13.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:12 smithi089 bash[82202]: audit 2024-07-15T07:00:11.651495+0000 mon.a (mon.0) 249 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:00:13.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:12 smithi089 bash[82202]: cephadm 2024-07-15T07:00:11.654743+0000 mgr.y (mgr.25244) 150 : cephadm [INF] Reconfiguring mds.foo.smithi134.wugtzh (monmap changed)... 2024-07-15T07:00:13.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:12 smithi089 bash[82202]: audit 2024-07-15T07:00:11.655393+0000 mon.a (mon.0) 250 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi134.wugtzh", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-07-15T07:00:13.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:12 smithi089 bash[82202]: audit 2024-07-15T07:00:11.657355+0000 mon.a (mon.0) 251 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T07:00:13.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:12 smithi089 bash[82202]: cephadm 2024-07-15T07:00:11.659316+0000 mgr.y (mgr.25244) 151 : cephadm [INF] Reconfiguring daemon mds.foo.smithi134.wugtzh on smithi134 2024-07-15T07:00:13.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:13 smithi134 bash[67624]: cluster 2024-07-15T07:00:12.771083+0000 mgr.y (mgr.25244) 152 : cluster [DBG] pgmap v97: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:00:13.805 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:00:13 smithi134 bash[60729]: ts=2024-07-15T07:00:13.508Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi134\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi134\", job=\"node\", machine=\"x86_64\", nodename=\"smithi134\", release=\"5.15.0-113-generic\", sysname=\"Linux\", version=\"#123-Ubuntu SMP Mon Jun 10 08:16:17 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", domainname=\"(none)\", instance=\"smithi134\", job=\"node\", machine=\"x86_64\", nodename=\"smithi134\", release=\"5.15.0-113-generic\", sysname=\"Linux\", version=\"#123-Ubuntu SMP Mon Jun 10 08:16:17 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T07:00:14.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:13 smithi089 bash[79810]: cluster 2024-07-15T07:00:12.771083+0000 mgr.y (mgr.25244) 152 : cluster [DBG] pgmap v97: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:00:14.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:13 smithi089 bash[82202]: cluster 2024-07-15T07:00:12.771083+0000 mgr.y (mgr.25244) 152 : cluster [DBG] pgmap v97: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:00:14.619 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 07:00:14 smithi089 bash[72894]: ::ffff:172.21.15.134 - - [15/Jul/2024:07:00:14] "GET /metrics HTTP/1.1" 200 33866 "" "Prometheus/2.43.0" 2024-07-15T07:00:14.804 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:00:14 smithi134 bash[60729]: ts=2024-07-15T07:00:14.453Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T07:00:14.805 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:00:14 smithi134 bash[60729]: ts=2024-07-15T07:00:14.454Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T07:00:15.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:14 smithi089 bash[79810]: audit 2024-07-15T07:00:13.858239+0000 mon.a (mon.0) 252 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:00:15.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:14 smithi089 bash[79810]: audit 2024-07-15T07:00:13.868406+0000 mon.a (mon.0) 253 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:00:15.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:14 smithi089 bash[79810]: audit 2024-07-15T07:00:13.959388+0000 mon.a (mon.0) 254 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T07:00:15.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:14 smithi089 bash[79810]: cluster 2024-07-15T07:00:14.771849+0000 mgr.y (mgr.25244) 153 : cluster [DBG] pgmap v98: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:00:15.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:14 smithi089 bash[82202]: audit 2024-07-15T07:00:13.858239+0000 mon.a (mon.0) 252 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:00:15.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:14 smithi089 bash[82202]: audit 2024-07-15T07:00:13.868406+0000 mon.a (mon.0) 253 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:00:15.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:14 smithi089 bash[82202]: audit 2024-07-15T07:00:13.959388+0000 mon.a (mon.0) 254 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T07:00:15.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:14 smithi089 bash[82202]: cluster 2024-07-15T07:00:14.771849+0000 mgr.y (mgr.25244) 153 : cluster [DBG] pgmap v98: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:00:15.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:14 smithi134 bash[67624]: audit 2024-07-15T07:00:13.858239+0000 mon.a (mon.0) 252 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:00:15.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:14 smithi134 bash[67624]: audit 2024-07-15T07:00:13.868406+0000 mon.a (mon.0) 253 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:00:15.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:14 smithi134 bash[67624]: audit 2024-07-15T07:00:13.959388+0000 mon.a (mon.0) 254 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T07:00:15.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:14 smithi134 bash[67624]: cluster 2024-07-15T07:00:14.771849+0000 mgr.y (mgr.25244) 153 : cluster [DBG] pgmap v98: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:00:17.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:16 smithi089 bash[79810]: cluster 2024-07-15T07:00:16.772949+0000 mgr.y (mgr.25244) 154 : cluster [DBG] pgmap v99: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:00:17.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:16 smithi089 bash[82202]: cluster 2024-07-15T07:00:16.772949+0000 mgr.y (mgr.25244) 154 : cluster [DBG] pgmap v99: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:00:17.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:16 smithi134 bash[67624]: cluster 2024-07-15T07:00:16.772949+0000 mgr.y (mgr.25244) 154 : cluster [DBG] pgmap v99: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:00:19.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:18 smithi134 bash[67624]: cluster 2024-07-15T07:00:18.773591+0000 mgr.y (mgr.25244) 155 : cluster [DBG] pgmap v100: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:00:19.055 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:00:18 smithi134 bash[60729]: ts=2024-07-15T07:00:18.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1102-g9a202024 (9a2020246f85463d4fdb115c94aa40b463caa1d2) quincy (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1102-g9a202024 (9a2020246f85463d4fdb115c94aa40b463caa1d2) quincy (stable)\", cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T07:00:19.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:18 smithi089 bash[79810]: cluster 2024-07-15T07:00:18.773591+0000 mgr.y (mgr.25244) 155 : cluster [DBG] pgmap v100: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:00:19.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:18 smithi089 bash[82202]: cluster 2024-07-15T07:00:18.773591+0000 mgr.y (mgr.25244) 155 : cluster [DBG] pgmap v100: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:00:20.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:20 smithi134 bash[67624]: audit 2024-07-15T07:00:19.545781+0000 mon.a (mon.0) 255 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:00:20.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:20 smithi134 bash[67624]: audit 2024-07-15T07:00:19.550745+0000 mon.a (mon.0) 256 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:00:20.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:20 smithi134 bash[67624]: audit 2024-07-15T07:00:19.787758+0000 mon.a (mon.0) 257 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:00:20.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:20 smithi134 bash[67624]: audit 2024-07-15T07:00:19.961290+0000 mon.a (mon.0) 258 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T07:00:20.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:20 smithi134 bash[67624]: audit 2024-07-15T07:00:19.962415+0000 mon.a (mon.0) 259 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T07:00:20.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:20 smithi134 bash[67624]: audit 2024-07-15T07:00:19.969960+0000 mon.a (mon.0) 260 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:00:20.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:20 smithi089 bash[79810]: audit 2024-07-15T07:00:19.545781+0000 mon.a (mon.0) 255 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:00:20.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:20 smithi089 bash[79810]: audit 2024-07-15T07:00:19.550745+0000 mon.a (mon.0) 256 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:00:20.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:20 smithi089 bash[79810]: audit 2024-07-15T07:00:19.787758+0000 mon.a (mon.0) 257 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:00:20.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:20 smithi089 bash[79810]: audit 2024-07-15T07:00:19.961290+0000 mon.a (mon.0) 258 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T07:00:20.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:20 smithi089 bash[79810]: audit 2024-07-15T07:00:19.962415+0000 mon.a (mon.0) 259 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T07:00:20.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:20 smithi089 bash[79810]: audit 2024-07-15T07:00:19.969960+0000 mon.a (mon.0) 260 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:00:20.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:20 smithi089 bash[82202]: audit 2024-07-15T07:00:19.545781+0000 mon.a (mon.0) 255 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:00:20.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:20 smithi089 bash[82202]: audit 2024-07-15T07:00:19.550745+0000 mon.a (mon.0) 256 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:00:20.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:20 smithi089 bash[82202]: audit 2024-07-15T07:00:19.787758+0000 mon.a (mon.0) 257 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:00:20.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:20 smithi089 bash[82202]: audit 2024-07-15T07:00:19.961290+0000 mon.a (mon.0) 258 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T07:00:20.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:20 smithi089 bash[82202]: audit 2024-07-15T07:00:19.962415+0000 mon.a (mon.0) 259 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T07:00:20.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:20 smithi089 bash[82202]: audit 2024-07-15T07:00:19.969960+0000 mon.a (mon.0) 260 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:00:21.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:21 smithi134 bash[67624]: cluster 2024-07-15T07:00:20.774556+0000 mgr.y (mgr.25244) 156 : cluster [DBG] pgmap v101: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:00:21.869 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:21 smithi089 bash[79810]: cluster 2024-07-15T07:00:20.774556+0000 mgr.y (mgr.25244) 156 : cluster [DBG] pgmap v101: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:00:21.870 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:21 smithi089 bash[82202]: cluster 2024-07-15T07:00:20.774556+0000 mgr.y (mgr.25244) 156 : cluster [DBG] pgmap v101: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:00:22.804 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:00:22 smithi134 bash[60729]: ts=2024-07-15T07:00:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T07:00:23.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:22 smithi089 bash[79810]: cluster 2024-07-15T07:00:22.775329+0000 mgr.y (mgr.25244) 157 : cluster [DBG] pgmap v102: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:00:23.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:22 smithi089 bash[82202]: cluster 2024-07-15T07:00:22.775329+0000 mgr.y (mgr.25244) 157 : cluster [DBG] pgmap v102: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:00:23.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:22 smithi134 bash[67624]: cluster 2024-07-15T07:00:22.775329+0000 mgr.y (mgr.25244) 157 : cluster [DBG] pgmap v102: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:00:23.804 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:00:23 smithi134 bash[60729]: ts=2024-07-15T07:00:23.508Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi134\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi134\", job=\"node\", machine=\"x86_64\", nodename=\"smithi134\", release=\"5.15.0-113-generic\", sysname=\"Linux\", version=\"#123-Ubuntu SMP Mon Jun 10 08:16:17 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", domainname=\"(none)\", instance=\"smithi134\", job=\"node\", machine=\"x86_64\", nodename=\"smithi134\", release=\"5.15.0-113-generic\", sysname=\"Linux\", version=\"#123-Ubuntu SMP Mon Jun 10 08:16:17 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T07:00:24.619 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 07:00:24 smithi089 bash[72894]: ::ffff:172.21.15.134 - - [15/Jul/2024:07:00:24] "GET /metrics HTTP/1.1" 200 33861 "" "Prometheus/2.43.0" 2024-07-15T07:00:24.805 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:00:24 smithi134 bash[60729]: ts=2024-07-15T07:00:24.453Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T07:00:24.812 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:00:24 smithi134 bash[60729]: ts=2024-07-15T07:00:24.454Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T07:00:25.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:24 smithi089 bash[79810]: cluster 2024-07-15T07:00:24.775964+0000 mgr.y (mgr.25244) 158 : cluster [DBG] pgmap v103: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:00:25.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:24 smithi089 bash[82202]: cluster 2024-07-15T07:00:24.775964+0000 mgr.y (mgr.25244) 158 : cluster [DBG] pgmap v103: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:00:25.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:24 smithi134 bash[67624]: cluster 2024-07-15T07:00:24.775964+0000 mgr.y (mgr.25244) 158 : cluster [DBG] pgmap v103: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:00:27.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:26 smithi089 bash[79810]: cluster 2024-07-15T07:00:26.777014+0000 mgr.y (mgr.25244) 159 : cluster [DBG] pgmap v104: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:00:27.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:26 smithi089 bash[82202]: cluster 2024-07-15T07:00:26.777014+0000 mgr.y (mgr.25244) 159 : cluster [DBG] pgmap v104: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:00:27.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:26 smithi134 bash[67624]: cluster 2024-07-15T07:00:26.777014+0000 mgr.y (mgr.25244) 159 : cluster [DBG] pgmap v104: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:00:29.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:28 smithi089 bash[79810]: cluster 2024-07-15T07:00:28.777642+0000 mgr.y (mgr.25244) 160 : cluster [DBG] pgmap v105: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:00:29.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:28 smithi089 bash[82202]: cluster 2024-07-15T07:00:28.777642+0000 mgr.y (mgr.25244) 160 : cluster [DBG] pgmap v105: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:00:29.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:28 smithi134 bash[67624]: cluster 2024-07-15T07:00:28.777642+0000 mgr.y (mgr.25244) 160 : cluster [DBG] pgmap v105: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:00:31.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:30 smithi089 bash[79810]: cluster 2024-07-15T07:00:30.778641+0000 mgr.y (mgr.25244) 161 : cluster [DBG] pgmap v106: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:00:31.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:30 smithi089 bash[82202]: cluster 2024-07-15T07:00:30.778641+0000 mgr.y (mgr.25244) 161 : cluster [DBG] pgmap v106: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:00:31.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:30 smithi134 bash[67624]: cluster 2024-07-15T07:00:30.778641+0000 mgr.y (mgr.25244) 161 : cluster [DBG] pgmap v106: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:00:32.784 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -e sha1=6ea7aaa9f93d34dc4d21862afe00140dfddbd706 -- bash -c 'ceph orch upgrade resume' 2024-07-15T07:00:32.804 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:00:32 smithi134 bash[60729]: ts=2024-07-15T07:00:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T07:00:33.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:32 smithi089 bash[82202]: cluster 2024-07-15T07:00:32.779173+0000 mgr.y (mgr.25244) 162 : cluster [DBG] pgmap v107: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:00:33.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:32 smithi089 bash[79810]: cluster 2024-07-15T07:00:32.779173+0000 mgr.y (mgr.25244) 162 : cluster [DBG] pgmap v107: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:00:33.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:32 smithi134 bash[67624]: cluster 2024-07-15T07:00:32.779173+0000 mgr.y (mgr.25244) 162 : cluster [DBG] pgmap v107: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:00:34.619 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 07:00:34 smithi089 bash[72894]: ::ffff:172.21.15.134 - - [15/Jul/2024:07:00:34] "GET /metrics HTTP/1.1" 200 33861 "" "Prometheus/2.43.0" 2024-07-15T07:00:35.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:34 smithi089 bash[79810]: cluster 2024-07-15T07:00:34.779779+0000 mgr.y (mgr.25244) 163 : cluster [DBG] pgmap v108: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:00:35.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:34 smithi089 bash[79810]: audit 2024-07-15T07:00:34.788255+0000 mon.a (mon.0) 261 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:00:35.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:34 smithi089 bash[82202]: cluster 2024-07-15T07:00:34.779779+0000 mgr.y (mgr.25244) 163 : cluster [DBG] pgmap v108: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:00:35.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:34 smithi089 bash[82202]: audit 2024-07-15T07:00:34.788255+0000 mon.a (mon.0) 261 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:00:35.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:34 smithi134 bash[67624]: cluster 2024-07-15T07:00:34.779779+0000 mgr.y (mgr.25244) 163 : cluster [DBG] pgmap v108: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:00:35.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:34 smithi134 bash[67624]: audit 2024-07-15T07:00:34.788255+0000 mon.a (mon.0) 261 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:00:37.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:36 smithi089 bash[79810]: cluster 2024-07-15T07:00:36.780826+0000 mgr.y (mgr.25244) 164 : cluster [DBG] pgmap v109: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:00:37.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:36 smithi089 bash[82202]: cluster 2024-07-15T07:00:36.780826+0000 mgr.y (mgr.25244) 164 : cluster [DBG] pgmap v109: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:00:37.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:36 smithi134 bash[67624]: cluster 2024-07-15T07:00:36.780826+0000 mgr.y (mgr.25244) 164 : cluster [DBG] pgmap v109: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:00:37.566 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T07:00:39.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:38 smithi089 bash[79810]: cluster 2024-07-15T07:00:38.781452+0000 mgr.y (mgr.25244) 165 : cluster [DBG] pgmap v110: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:00:39.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:38 smithi089 bash[82202]: cluster 2024-07-15T07:00:38.781452+0000 mgr.y (mgr.25244) 165 : cluster [DBG] pgmap v110: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:00:39.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:38 smithi134 bash[67624]: cluster 2024-07-15T07:00:38.781452+0000 mgr.y (mgr.25244) 165 : cluster [DBG] pgmap v110: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:00:41.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:40 smithi089 bash[79810]: cluster 2024-07-15T07:00:40.782464+0000 mgr.y (mgr.25244) 166 : cluster [DBG] pgmap v111: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:00:41.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:40 smithi089 bash[82202]: cluster 2024-07-15T07:00:40.782464+0000 mgr.y (mgr.25244) 166 : cluster [DBG] pgmap v111: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:00:41.259 INFO:teuthology.orchestra.run.smithi089.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:badc66b92d2a2cfbbce7f293b5accdc50f486d2b2d966876e46fe28d05f464f1 2024-07-15T07:00:41.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:40 smithi134 bash[67624]: cluster 2024-07-15T07:00:40.782464+0000 mgr.y (mgr.25244) 166 : cluster [DBG] pgmap v111: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:00:42.016 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -e sha1=6ea7aaa9f93d34dc4d21862afe00140dfddbd706 -- bash -c 'echo wait for all mons to upgrade' 2024-07-15T07:00:42.369 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:42 smithi089 bash[82202]: audit 2024-07-15T07:00:41.258196+0000 mgr.y (mgr.25244) 167 : audit [DBG] from='client.44120 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T07:00:42.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:42 smithi089 bash[82202]: cephadm 2024-07-15T07:00:41.258664+0000 mgr.y (mgr.25244) 168 : cephadm [INF] Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:badc66b92d2a2cfbbce7f293b5accdc50f486d2b2d966876e46fe28d05f464f1 2024-07-15T07:00:42.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:42 smithi089 bash[82202]: audit 2024-07-15T07:00:41.264378+0000 mon.a (mon.0) 262 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:00:42.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:42 smithi089 bash[82202]: audit 2024-07-15T07:00:41.265431+0000 mon.a (mon.0) 263 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T07:00:42.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:42 smithi089 bash[79810]: audit 2024-07-15T07:00:41.258196+0000 mgr.y (mgr.25244) 167 : audit [DBG] from='client.44120 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T07:00:42.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:42 smithi089 bash[79810]: cephadm 2024-07-15T07:00:41.258664+0000 mgr.y (mgr.25244) 168 : cephadm [INF] Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:badc66b92d2a2cfbbce7f293b5accdc50f486d2b2d966876e46fe28d05f464f1 2024-07-15T07:00:42.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:42 smithi089 bash[79810]: audit 2024-07-15T07:00:41.264378+0000 mon.a (mon.0) 262 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:00:42.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:42 smithi089 bash[79810]: audit 2024-07-15T07:00:41.265431+0000 mon.a (mon.0) 263 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T07:00:42.554 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:00:42 smithi134 bash[60729]: ts=2024-07-15T07:00:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T07:00:42.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:42 smithi134 bash[67624]: audit 2024-07-15T07:00:41.258196+0000 mgr.y (mgr.25244) 167 : audit [DBG] from='client.44120 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T07:00:42.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:42 smithi134 bash[67624]: cephadm 2024-07-15T07:00:41.258664+0000 mgr.y (mgr.25244) 168 : cephadm [INF] Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:badc66b92d2a2cfbbce7f293b5accdc50f486d2b2d966876e46fe28d05f464f1 2024-07-15T07:00:42.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:42 smithi134 bash[67624]: audit 2024-07-15T07:00:41.264378+0000 mon.a (mon.0) 262 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:00:42.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:42 smithi134 bash[67624]: audit 2024-07-15T07:00:41.265431+0000 mon.a (mon.0) 263 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T07:00:43.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:43 smithi134 bash[67624]: cluster 2024-07-15T07:00:42.783088+0000 mgr.y (mgr.25244) 169 : cluster [DBG] pgmap v112: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:00:43.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:43 smithi089 bash[79810]: cluster 2024-07-15T07:00:42.783088+0000 mgr.y (mgr.25244) 169 : cluster [DBG] pgmap v112: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:00:43.619 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:43 smithi089 bash[82202]: cluster 2024-07-15T07:00:42.783088+0000 mgr.y (mgr.25244) 169 : cluster [DBG] pgmap v112: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:00:44.619 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 07:00:44 smithi089 bash[72894]: ::ffff:172.21.15.134 - - [15/Jul/2024:07:00:44] "GET /metrics HTTP/1.1" 200 33861 "" "Prometheus/2.43.0" 2024-07-15T07:00:45.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:44 smithi089 bash[79810]: cluster 2024-07-15T07:00:44.785756+0000 mgr.y (mgr.25244) 170 : cluster [DBG] pgmap v113: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:00:45.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:44 smithi089 bash[82202]: cluster 2024-07-15T07:00:44.785756+0000 mgr.y (mgr.25244) 170 : cluster [DBG] pgmap v113: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:00:45.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:44 smithi134 bash[67624]: cluster 2024-07-15T07:00:44.785756+0000 mgr.y (mgr.25244) 170 : cluster [DBG] pgmap v113: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:00:46.802 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T07:00:47.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:46 smithi089 bash[79810]: cluster 2024-07-15T07:00:46.786759+0000 mgr.y (mgr.25244) 171 : cluster [DBG] pgmap v114: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:00:47.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:46 smithi089 bash[82202]: cluster 2024-07-15T07:00:46.786759+0000 mgr.y (mgr.25244) 171 : cluster [DBG] pgmap v114: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:00:47.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:46 smithi134 bash[67624]: cluster 2024-07-15T07:00:46.786759+0000 mgr.y (mgr.25244) 171 : cluster [DBG] pgmap v114: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:00:48.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:48 smithi089 bash[79810]: audit 2024-07-15T07:00:47.105844+0000 mon.a (mon.0) 264 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:00:48.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:48 smithi089 bash[79810]: audit 2024-07-15T07:00:47.115735+0000 mon.a (mon.0) 265 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:00:48.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:48 smithi089 bash[79810]: audit 2024-07-15T07:00:47.598670+0000 mon.a (mon.0) 266 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T07:00:48.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:48 smithi089 bash[79810]: audit 2024-07-15T07:00:47.600176+0000 mon.a (mon.0) 267 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T07:00:48.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:48 smithi089 bash[79810]: audit 2024-07-15T07:00:47.609973+0000 mon.a (mon.0) 268 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:00:48.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:48 smithi089 bash[79810]: audit 2024-07-15T07:00:47.744535+0000 mon.a (mon.0) 269 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T07:00:48.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:48 smithi089 bash[79810]: audit 2024-07-15T07:00:47.748196+0000 mon.a (mon.0) 270 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-15T07:00:48.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:48 smithi089 bash[79810]: audit 2024-07-15T07:00:47.750113+0000 mon.a (mon.0) 271 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-15T07:00:48.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:48 smithi089 bash[79810]: cephadm 2024-07-15T07:00:47.750933+0000 mgr.y (mgr.25244) 172 : cephadm [INF] Upgrade: Setting container_image for all mon 2024-07-15T07:00:48.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:48 smithi089 bash[79810]: audit 2024-07-15T07:00:47.759679+0000 mon.a (mon.0) 272 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:00:48.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:48 smithi089 bash[79810]: audit 2024-07-15T07:00:47.761011+0000 mon.a (mon.0) 273 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-07-15T07:00:48.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:48 smithi089 bash[79810]: audit 2024-07-15T07:00:47.769106+0000 mon.a (mon.0) 274 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-07-15T07:00:48.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:48 smithi089 bash[79810]: audit 2024-07-15T07:00:47.770519+0000 mon.a (mon.0) 275 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-07-15T07:00:48.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:48 smithi089 bash[79810]: audit 2024-07-15T07:00:47.779320+0000 mon.a (mon.0) 276 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-07-15T07:00:48.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:48 smithi089 bash[79810]: audit 2024-07-15T07:00:47.780788+0000 mon.a (mon.0) 277 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-07-15T07:00:48.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:48 smithi089 bash[79810]: audit 2024-07-15T07:00:47.789290+0000 mon.a (mon.0) 278 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-07-15T07:00:48.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:48 smithi089 bash[79810]: audit 2024-07-15T07:00:47.791546+0000 mon.a (mon.0) 279 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-15T07:00:48.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:48 smithi089 bash[79810]: cephadm 2024-07-15T07:00:47.793084+0000 mgr.y (mgr.25244) 173 : cephadm [INF] Upgrade: Setting container_image for all crash 2024-07-15T07:00:48.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:48 smithi089 bash[79810]: audit 2024-07-15T07:00:47.802352+0000 mon.a (mon.0) 280 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:00:48.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:48 smithi089 bash[79810]: audit 2024-07-15T07:00:47.805036+0000 mon.a (mon.0) 281 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-07-15T07:00:48.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:48 smithi089 bash[79810]: audit 2024-07-15T07:00:47.805296+0000 mgr.y (mgr.25244) 174 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-07-15T07:00:48.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:48 smithi089 bash[79810]: cephadm 2024-07-15T07:00:47.806893+0000 mgr.y (mgr.25244) 175 : cephadm [INF] Upgrade: osd.0 is safe to restart 2024-07-15T07:00:48.372 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:48 smithi089 bash[82202]: audit 2024-07-15T07:00:47.105844+0000 mon.a (mon.0) 264 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:00:48.372 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:48 smithi089 bash[82202]: audit 2024-07-15T07:00:47.115735+0000 mon.a (mon.0) 265 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:00:48.372 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:48 smithi089 bash[82202]: audit 2024-07-15T07:00:47.598670+0000 mon.a (mon.0) 266 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T07:00:48.372 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:48 smithi089 bash[82202]: audit 2024-07-15T07:00:47.600176+0000 mon.a (mon.0) 267 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T07:00:48.372 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:48 smithi089 bash[82202]: audit 2024-07-15T07:00:47.609973+0000 mon.a (mon.0) 268 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:00:48.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:48 smithi089 bash[82202]: audit 2024-07-15T07:00:47.744535+0000 mon.a (mon.0) 269 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T07:00:48.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:48 smithi089 bash[82202]: audit 2024-07-15T07:00:47.748196+0000 mon.a (mon.0) 270 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-15T07:00:48.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:48 smithi089 bash[82202]: audit 2024-07-15T07:00:47.750113+0000 mon.a (mon.0) 271 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-15T07:00:48.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:48 smithi089 bash[82202]: cephadm 2024-07-15T07:00:47.750933+0000 mgr.y (mgr.25244) 172 : cephadm [INF] Upgrade: Setting container_image for all mon 2024-07-15T07:00:48.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:48 smithi089 bash[82202]: audit 2024-07-15T07:00:47.759679+0000 mon.a (mon.0) 272 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:00:48.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:48 smithi089 bash[82202]: audit 2024-07-15T07:00:47.761011+0000 mon.a (mon.0) 273 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-07-15T07:00:48.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:48 smithi089 bash[82202]: audit 2024-07-15T07:00:47.769106+0000 mon.a (mon.0) 274 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-07-15T07:00:48.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:48 smithi089 bash[82202]: audit 2024-07-15T07:00:47.770519+0000 mon.a (mon.0) 275 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-07-15T07:00:48.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:48 smithi089 bash[82202]: audit 2024-07-15T07:00:47.779320+0000 mon.a (mon.0) 276 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-07-15T07:00:48.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:48 smithi089 bash[82202]: audit 2024-07-15T07:00:47.780788+0000 mon.a (mon.0) 277 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-07-15T07:00:48.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:48 smithi089 bash[82202]: audit 2024-07-15T07:00:47.789290+0000 mon.a (mon.0) 278 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-07-15T07:00:48.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:48 smithi089 bash[82202]: audit 2024-07-15T07:00:47.791546+0000 mon.a (mon.0) 279 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-15T07:00:48.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:48 smithi089 bash[82202]: cephadm 2024-07-15T07:00:47.793084+0000 mgr.y (mgr.25244) 173 : cephadm [INF] Upgrade: Setting container_image for all crash 2024-07-15T07:00:48.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:48 smithi089 bash[82202]: audit 2024-07-15T07:00:47.802352+0000 mon.a (mon.0) 280 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:00:48.374 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:48 smithi089 bash[82202]: audit 2024-07-15T07:00:47.805036+0000 mon.a (mon.0) 281 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-07-15T07:00:48.374 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:48 smithi089 bash[82202]: audit 2024-07-15T07:00:47.805296+0000 mgr.y (mgr.25244) 174 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-07-15T07:00:48.374 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:48 smithi089 bash[82202]: cephadm 2024-07-15T07:00:47.806893+0000 mgr.y (mgr.25244) 175 : cephadm [INF] Upgrade: osd.0 is safe to restart 2024-07-15T07:00:48.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:48 smithi134 bash[67624]: audit 2024-07-15T07:00:47.105844+0000 mon.a (mon.0) 264 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:00:48.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:48 smithi134 bash[67624]: audit 2024-07-15T07:00:47.115735+0000 mon.a (mon.0) 265 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:00:48.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:48 smithi134 bash[67624]: audit 2024-07-15T07:00:47.598670+0000 mon.a (mon.0) 266 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T07:00:48.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:48 smithi134 bash[67624]: audit 2024-07-15T07:00:47.600176+0000 mon.a (mon.0) 267 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T07:00:48.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:48 smithi134 bash[67624]: audit 2024-07-15T07:00:47.609973+0000 mon.a (mon.0) 268 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:00:48.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:48 smithi134 bash[67624]: audit 2024-07-15T07:00:47.744535+0000 mon.a (mon.0) 269 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T07:00:48.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:48 smithi134 bash[67624]: audit 2024-07-15T07:00:47.748196+0000 mon.a (mon.0) 270 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-15T07:00:48.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:48 smithi134 bash[67624]: audit 2024-07-15T07:00:47.750113+0000 mon.a (mon.0) 271 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-15T07:00:48.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:48 smithi134 bash[67624]: cephadm 2024-07-15T07:00:47.750933+0000 mgr.y (mgr.25244) 172 : cephadm [INF] Upgrade: Setting container_image for all mon 2024-07-15T07:00:48.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:48 smithi134 bash[67624]: audit 2024-07-15T07:00:47.759679+0000 mon.a (mon.0) 272 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:00:48.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:48 smithi134 bash[67624]: audit 2024-07-15T07:00:47.761011+0000 mon.a (mon.0) 273 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-07-15T07:00:48.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:48 smithi134 bash[67624]: audit 2024-07-15T07:00:47.769106+0000 mon.a (mon.0) 274 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-07-15T07:00:48.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:48 smithi134 bash[67624]: audit 2024-07-15T07:00:47.770519+0000 mon.a (mon.0) 275 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-07-15T07:00:48.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:48 smithi134 bash[67624]: audit 2024-07-15T07:00:47.779320+0000 mon.a (mon.0) 276 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-07-15T07:00:48.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:48 smithi134 bash[67624]: audit 2024-07-15T07:00:47.780788+0000 mon.a (mon.0) 277 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-07-15T07:00:48.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:48 smithi134 bash[67624]: audit 2024-07-15T07:00:47.789290+0000 mon.a (mon.0) 278 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-07-15T07:00:48.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:48 smithi134 bash[67624]: audit 2024-07-15T07:00:47.791546+0000 mon.a (mon.0) 279 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-15T07:00:48.557 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:48 smithi134 bash[67624]: cephadm 2024-07-15T07:00:47.793084+0000 mgr.y (mgr.25244) 173 : cephadm [INF] Upgrade: Setting container_image for all crash 2024-07-15T07:00:48.557 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:48 smithi134 bash[67624]: audit 2024-07-15T07:00:47.802352+0000 mon.a (mon.0) 280 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:00:48.557 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:48 smithi134 bash[67624]: audit 2024-07-15T07:00:47.805036+0000 mon.a (mon.0) 281 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-07-15T07:00:48.557 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:48 smithi134 bash[67624]: audit 2024-07-15T07:00:47.805296+0000 mgr.y (mgr.25244) 174 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-07-15T07:00:48.557 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:48 smithi134 bash[67624]: cephadm 2024-07-15T07:00:47.806893+0000 mgr.y (mgr.25244) 175 : cephadm [INF] Upgrade: osd.0 is safe to restart 2024-07-15T07:00:49.369 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:49 smithi089 bash[79810]: cluster 2024-07-15T07:00:48.787420+0000 mgr.y (mgr.25244) 176 : cluster [DBG] pgmap v115: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:00:49.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:49 smithi089 bash[82202]: cluster 2024-07-15T07:00:48.787420+0000 mgr.y (mgr.25244) 176 : cluster [DBG] pgmap v115: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:00:49.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:49 smithi134 bash[67624]: cluster 2024-07-15T07:00:48.787420+0000 mgr.y (mgr.25244) 176 : cluster [DBG] pgmap v115: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:00:50.252 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:50 smithi089 bash[82202]: audit 2024-07-15T07:00:49.788967+0000 mon.a (mon.0) 282 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:00:50.252 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:50 smithi089 bash[79810]: audit 2024-07-15T07:00:49.788967+0000 mon.a (mon.0) 282 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:00:50.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:50 smithi134 bash[67624]: audit 2024-07-15T07:00:49.788967+0000 mon.a (mon.0) 282 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:00:51.369 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:51 smithi089 bash[79810]: cluster 2024-07-15T07:00:50.788414+0000 mgr.y (mgr.25244) 177 : cluster [DBG] pgmap v116: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:00:51.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:51 smithi089 bash[82202]: cluster 2024-07-15T07:00:50.788414+0000 mgr.y (mgr.25244) 177 : cluster [DBG] pgmap v116: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:00:51.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:51 smithi134 bash[67624]: cluster 2024-07-15T07:00:50.788414+0000 mgr.y (mgr.25244) 177 : cluster [DBG] pgmap v116: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:00:51.609 INFO:teuthology.orchestra.run.smithi089.stdout:wait for all mons to upgrade 2024-07-15T07:00:52.804 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:00:52 smithi134 bash[60729]: ts=2024-07-15T07:00:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T07:00:52.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:52 smithi134 bash[67624]: cephadm 2024-07-15T07:00:51.441852+0000 mgr.y (mgr.25244) 178 : cephadm [INF] Upgrade: Updating osd.0 2024-07-15T07:00:52.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:52 smithi134 bash[67624]: audit 2024-07-15T07:00:51.455806+0000 mon.a (mon.0) 283 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:00:52.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:52 smithi134 bash[67624]: audit 2024-07-15T07:00:51.457552+0000 mon.a (mon.0) 284 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-07-15T07:00:52.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:52 smithi134 bash[67624]: audit 2024-07-15T07:00:51.458971+0000 mon.a (mon.0) 285 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T07:00:52.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:52 smithi134 bash[67624]: cephadm 2024-07-15T07:00:51.463229+0000 mgr.y (mgr.25244) 179 : cephadm [INF] Deploying daemon osd.0 on smithi089 2024-07-15T07:00:52.859 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:52 smithi089 bash[82202]: cephadm 2024-07-15T07:00:51.441852+0000 mgr.y (mgr.25244) 178 : cephadm [INF] Upgrade: Updating osd.0 2024-07-15T07:00:52.859 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:52 smithi089 bash[82202]: audit 2024-07-15T07:00:51.455806+0000 mon.a (mon.0) 283 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:00:52.859 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:52 smithi089 bash[82202]: audit 2024-07-15T07:00:51.457552+0000 mon.a (mon.0) 284 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-07-15T07:00:52.859 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:52 smithi089 bash[82202]: audit 2024-07-15T07:00:51.458971+0000 mon.a (mon.0) 285 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T07:00:52.859 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:52 smithi089 bash[82202]: cephadm 2024-07-15T07:00:51.463229+0000 mgr.y (mgr.25244) 179 : cephadm [INF] Deploying daemon osd.0 on smithi089 2024-07-15T07:00:52.859 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:52 smithi089 bash[79810]: cephadm 2024-07-15T07:00:51.441852+0000 mgr.y (mgr.25244) 178 : cephadm [INF] Upgrade: Updating osd.0 2024-07-15T07:00:52.859 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:52 smithi089 bash[79810]: audit 2024-07-15T07:00:51.455806+0000 mon.a (mon.0) 283 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:00:52.860 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:52 smithi089 bash[79810]: audit 2024-07-15T07:00:51.457552+0000 mon.a (mon.0) 284 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-07-15T07:00:52.860 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:52 smithi089 bash[79810]: audit 2024-07-15T07:00:51.458971+0000 mon.a (mon.0) 285 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T07:00:52.860 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:52 smithi089 bash[79810]: cephadm 2024-07-15T07:00:51.463229+0000 mgr.y (mgr.25244) 179 : cephadm [INF] Deploying daemon osd.0 on smithi089 2024-07-15T07:00:52.861 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -e sha1=6ea7aaa9f93d34dc4d21862afe00140dfddbd706 -- bash -c 'while ! ceph mon versions | grep $sha1 | grep '"'"': 3'"'"' ; do sleep 2 ; done' 2024-07-15T07:00:53.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:53 smithi134 bash[67624]: cluster 2024-07-15T07:00:52.789004+0000 mgr.y (mgr.25244) 180 : cluster [DBG] pgmap v117: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:00:53.854 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:53 smithi089 bash[79810]: cluster 2024-07-15T07:00:52.789004+0000 mgr.y (mgr.25244) 180 : cluster [DBG] pgmap v117: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:00:53.854 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:53 smithi089 bash[82202]: cluster 2024-07-15T07:00:52.789004+0000 mgr.y (mgr.25244) 180 : cluster [DBG] pgmap v117: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:00:54.623 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 07:00:54 smithi089 bash[72894]: ::ffff:172.21.15.134 - - [15/Jul/2024:07:00:54] "GET /metrics HTTP/1.1" 200 33865 "" "Prometheus/2.43.0" 2024-07-15T07:00:55.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:54 smithi089 bash[79810]: cluster 2024-07-15T07:00:54.789587+0000 mgr.y (mgr.25244) 181 : cluster [DBG] pgmap v118: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:00:55.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:54 smithi089 bash[82202]: cluster 2024-07-15T07:00:54.789587+0000 mgr.y (mgr.25244) 181 : cluster [DBG] pgmap v118: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:00:55.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:54 smithi134 bash[67624]: cluster 2024-07-15T07:00:54.789587+0000 mgr.y (mgr.25244) 181 : cluster [DBG] pgmap v118: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:00:57.083 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:56 smithi089 bash[79810]: cluster 2024-07-15T07:00:56.790708+0000 mgr.y (mgr.25244) 182 : cluster [DBG] pgmap v119: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:00:57.083 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:56 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:00:57.083 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 07:00:56 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:00:57.084 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:56 smithi089 bash[82202]: cluster 2024-07-15T07:00:56.790708+0000 mgr.y (mgr.25244) 182 : cluster [DBG] pgmap v119: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:00:57.084 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:56 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:00:57.084 INFO:journalctl@ceph.osd.0.smithi089.stdout:Jul 15 07:00:56 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:00:57.084 INFO:journalctl@ceph.osd.1.smithi089.stdout:Jul 15 07:00:56 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:00:57.085 INFO:journalctl@ceph.osd.3.smithi089.stdout:Jul 15 07:00:56 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:00:57.085 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 07:00:56 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:00:57.085 INFO:journalctl@ceph.osd.2.smithi089.stdout:Jul 15 07:00:56 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:00:57.086 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 07:00:56 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:00:57.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:56 smithi134 bash[67624]: cluster 2024-07-15T07:00:56.790708+0000 mgr.y (mgr.25244) 182 : cluster [DBG] pgmap v119: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:00:57.369 INFO:journalctl@ceph.osd.0.smithi089.stdout:Jul 15 07:00:57 smithi089 systemd[1]: Stopping Ceph osd.0 for 75ba02bc-4274-11ef-bcad-c7b262605968... 2024-07-15T07:00:57.370 INFO:journalctl@ceph.osd.0.smithi089.stdout:Jul 15 07:00:57 smithi089 bash[34386]: debug 2024-07-15T07:00:57.222+0000 7fe00ba19640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-07-15T07:00:57.370 INFO:journalctl@ceph.osd.0.smithi089.stdout:Jul 15 07:00:57 smithi089 bash[34386]: debug 2024-07-15T07:00:57.222+0000 7fe00ba19640 -1 osd.0 79 *** Got signal Terminated *** 2024-07-15T07:00:57.370 INFO:journalctl@ceph.osd.0.smithi089.stdout:Jul 15 07:00:57 smithi089 bash[34386]: debug 2024-07-15T07:00:57.222+0000 7fe00ba19640 -1 osd.0 79 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-07-15T07:00:58.133 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:57 smithi089 bash[82202]: cluster 2024-07-15T07:00:57.225722+0000 mon.a (mon.0) 286 : cluster [INF] osd.0 marked itself down and dead 2024-07-15T07:00:58.133 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:57 smithi089 bash[79810]: debug 2024-07-15T07:00:57.874+0000 7fb563a0a640 -1 log_channel(cluster) log [ERR] : Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-07-15T07:00:58.133 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:57 smithi089 bash[79810]: cluster 2024-07-15T07:00:57.225722+0000 mon.a (mon.0) 286 : cluster [INF] osd.0 marked itself down and dead 2024-07-15T07:00:58.199 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T07:00:58.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:57 smithi134 bash[67624]: cluster 2024-07-15T07:00:57.225722+0000 mon.a (mon.0) 286 : cluster [INF] osd.0 marked itself down and dead 2024-07-15T07:00:59.195 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:58 smithi089 bash[79810]: cluster 2024-07-15T07:00:57.878087+0000 mon.a (mon.0) 287 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-07-15T07:00:59.195 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:58 smithi089 bash[79810]: cluster 2024-07-15T07:00:57.878149+0000 mon.a (mon.0) 288 : cluster [ERR] Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-07-15T07:00:59.196 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:58 smithi089 bash[79810]: cluster 2024-07-15T07:00:57.898585+0000 mon.a (mon.0) 289 : cluster [DBG] osdmap e80: 8 total, 7 up, 8 in 2024-07-15T07:00:59.196 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:58 smithi089 bash[79810]: cluster 2024-07-15T07:00:58.791350+0000 mgr.y (mgr.25244) 183 : cluster [DBG] pgmap v121: 73 pgs: 11 stale+active+clean, 62 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:00:59.196 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:58 smithi089 bash[82202]: cluster 2024-07-15T07:00:57.878087+0000 mon.a (mon.0) 287 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-07-15T07:00:59.196 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:58 smithi089 bash[82202]: cluster 2024-07-15T07:00:57.878149+0000 mon.a (mon.0) 288 : cluster [ERR] Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-07-15T07:00:59.196 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:58 smithi089 bash[82202]: cluster 2024-07-15T07:00:57.898585+0000 mon.a (mon.0) 289 : cluster [DBG] osdmap e80: 8 total, 7 up, 8 in 2024-07-15T07:00:59.196 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:58 smithi089 bash[82202]: cluster 2024-07-15T07:00:58.791350+0000 mgr.y (mgr.25244) 183 : cluster [DBG] pgmap v121: 73 pgs: 11 stale+active+clean, 62 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:00:59.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:58 smithi134 bash[67624]: cluster 2024-07-15T07:00:57.878087+0000 mon.a (mon.0) 287 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-07-15T07:00:59.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:58 smithi134 bash[67624]: cluster 2024-07-15T07:00:57.878149+0000 mon.a (mon.0) 288 : cluster [ERR] Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-07-15T07:00:59.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:58 smithi134 bash[67624]: cluster 2024-07-15T07:00:57.898585+0000 mon.a (mon.0) 289 : cluster [DBG] osdmap e80: 8 total, 7 up, 8 in 2024-07-15T07:00:59.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:58 smithi134 bash[67624]: cluster 2024-07-15T07:00:58.791350+0000 mgr.y (mgr.25244) 183 : cluster [DBG] pgmap v121: 73 pgs: 11 stale+active+clean, 62 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:00:59.869 INFO:journalctl@ceph.osd.0.smithi089.stdout:Jul 15 07:00:59 smithi089 bash[87951]: ceph-75ba02bc-4274-11ef-bcad-c7b262605968-osd-0 2024-07-15T07:01:00.154 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:00:59 smithi089 bash[82202]: cluster 2024-07-15T07:00:58.908612+0000 mon.a (mon.0) 290 : cluster [DBG] osdmap e81: 8 total, 7 up, 8 in 2024-07-15T07:01:00.154 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:00:59 smithi089 bash[79810]: cluster 2024-07-15T07:00:58.908612+0000 mon.a (mon.0) 290 : cluster [DBG] osdmap e81: 8 total, 7 up, 8 in 2024-07-15T07:01:00.155 INFO:journalctl@ceph.osd.0.smithi089.stdout:Jul 15 07:00:59 smithi089 bash[88028]: Error response from daemon: No such container: ceph-75ba02bc-4274-11ef-bcad-c7b262605968-osd-0 2024-07-15T07:01:00.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:00:59 smithi134 bash[67624]: cluster 2024-07-15T07:00:58.908612+0000 mon.a (mon.0) 290 : cluster [DBG] osdmap e81: 8 total, 7 up, 8 in 2024-07-15T07:01:01.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:00 smithi134 bash[67624]: cluster 2024-07-15T07:01:00.791978+0000 mgr.y (mgr.25244) 184 : cluster [DBG] pgmap v123: 73 pgs: 19 active+undersized, 7 active+undersized+degraded, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2024-07-15T07:01:01.369 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:00 smithi089 bash[79810]: cluster 2024-07-15T07:01:00.791978+0000 mgr.y (mgr.25244) 184 : cluster [DBG] pgmap v123: 73 pgs: 19 active+undersized, 7 active+undersized+degraded, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2024-07-15T07:01:01.369 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:00 smithi089 bash[82202]: cluster 2024-07-15T07:01:00.791978+0000 mgr.y (mgr.25244) 184 : cluster [DBG] pgmap v123: 73 pgs: 19 active+undersized, 7 active+undersized+degraded, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2024-07-15T07:01:02.215 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:01 smithi089 bash[82202]: cluster 2024-07-15T07:01:00.902769+0000 mon.a (mon.0) 291 : cluster [WRN] Health check failed: Degraded data redundancy: 12/72 objects degraded (16.667%), 7 pgs degraded (PG_DEGRADED) 2024-07-15T07:01:02.216 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:01 smithi089 bash[79810]: cluster 2024-07-15T07:01:00.902769+0000 mon.a (mon.0) 291 : cluster [WRN] Health check failed: Degraded data redundancy: 12/72 objects degraded (16.667%), 7 pgs degraded (PG_DEGRADED) 2024-07-15T07:01:02.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:01 smithi134 bash[67624]: cluster 2024-07-15T07:01:00.902769+0000 mon.a (mon.0) 291 : cluster [WRN] Health check failed: Degraded data redundancy: 12/72 objects degraded (16.667%), 7 pgs degraded (PG_DEGRADED) 2024-07-15T07:01:02.804 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:01:02 smithi134 bash[60729]: ts=2024-07-15T07:01:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T07:01:03.286 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:02 smithi089 bash[82202]: cluster 2024-07-15T07:01:02.792460+0000 mgr.y (mgr.25244) 185 : cluster [DBG] pgmap v124: 73 pgs: 19 active+undersized, 7 active+undersized+degraded, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2024-07-15T07:01:03.286 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:02 smithi089 bash[79810]: cluster 2024-07-15T07:01:02.792460+0000 mgr.y (mgr.25244) 185 : cluster [DBG] pgmap v124: 73 pgs: 19 active+undersized, 7 active+undersized+degraded, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2024-07-15T07:01:03.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:02 smithi134 bash[67624]: cluster 2024-07-15T07:01:02.792460+0000 mgr.y (mgr.25244) 185 : cluster [DBG] pgmap v124: 73 pgs: 19 active+undersized, 7 active+undersized+degraded, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2024-07-15T07:01:04.119 INFO:journalctl@ceph.osd.0.smithi089.stdout:Jul 15 07:01:03 smithi089 systemd[1]: ceph-75ba02bc-4274-11ef-bcad-c7b262605968@osd.0.service: Deactivated successfully. 2024-07-15T07:01:04.119 INFO:journalctl@ceph.osd.0.smithi089.stdout:Jul 15 07:01:03 smithi089 systemd[1]: Stopped Ceph osd.0 for 75ba02bc-4274-11ef-bcad-c7b262605968. 2024-07-15T07:01:04.120 INFO:journalctl@ceph.osd.0.smithi089.stdout:Jul 15 07:01:04 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:01:04.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:04 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:01:04.120 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 07:01:04 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:01:04.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:04 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:01:04.121 INFO:journalctl@ceph.osd.1.smithi089.stdout:Jul 15 07:01:04 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:01:04.121 INFO:journalctl@ceph.osd.3.smithi089.stdout:Jul 15 07:01:04 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:01:04.121 INFO:journalctl@ceph.osd.2.smithi089.stdout:Jul 15 07:01:04 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:01:04.122 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 07:01:04 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:01:04.122 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 07:01:04 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:01:04.539 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 18.2.2-1609-g6ea7aaa9 (6ea7aaa9f93d34dc4d21862afe00140dfddbd706) reef (stable)": 3 2024-07-15T07:01:04.540 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 07:01:04 smithi089 bash[72894]: ::ffff:172.21.15.134 - - [15/Jul/2024:07:01:04] "GET /metrics HTTP/1.1" 200 33865 "" "Prometheus/2.43.0" 2024-07-15T07:01:04.540 INFO:journalctl@ceph.osd.0.smithi089.stdout:Jul 15 07:01:04 smithi089 systemd[1]: Started Ceph osd.0 for 75ba02bc-4274-11ef-bcad-c7b262605968. 2024-07-15T07:01:05.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:05 smithi089 bash[79810]: audit 2024-07-15T07:01:04.179744+0000 mon.a (mon.0) 292 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:01:05.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:05 smithi089 bash[79810]: audit 2024-07-15T07:01:04.185457+0000 mon.a (mon.0) 293 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:01:05.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:05 smithi089 bash[79810]: audit 2024-07-15T07:01:04.186684+0000 mon.a (mon.0) 294 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T07:01:05.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:05 smithi089 bash[79810]: audit 2024-07-15T07:01:04.532520+0000 mon.c (mon.2) 3 : audit [DBG] from='client.? 172.21.15.89:0/2559401453' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T07:01:05.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:05 smithi089 bash[79810]: cluster 2024-07-15T07:01:04.793149+0000 mgr.y (mgr.25244) 186 : cluster [DBG] pgmap v125: 73 pgs: 19 active+undersized, 7 active+undersized+degraded, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2024-07-15T07:01:05.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:05 smithi089 bash[79810]: audit 2024-07-15T07:01:04.795436+0000 mon.a (mon.0) 295 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:01:05.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:05 smithi089 bash[79810]: audit 2024-07-15T07:01:04.797264+0000 mon.a (mon.0) 296 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:01:05.371 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:05 smithi089 bash[82202]: audit 2024-07-15T07:01:04.179744+0000 mon.a (mon.0) 292 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:01:05.371 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:05 smithi089 bash[82202]: audit 2024-07-15T07:01:04.185457+0000 mon.a (mon.0) 293 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:01:05.371 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:05 smithi089 bash[82202]: audit 2024-07-15T07:01:04.186684+0000 mon.a (mon.0) 294 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T07:01:05.371 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:05 smithi089 bash[82202]: audit 2024-07-15T07:01:04.532520+0000 mon.c (mon.2) 3 : audit [DBG] from='client.? 172.21.15.89:0/2559401453' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T07:01:05.371 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:05 smithi089 bash[82202]: cluster 2024-07-15T07:01:04.793149+0000 mgr.y (mgr.25244) 186 : cluster [DBG] pgmap v125: 73 pgs: 19 active+undersized, 7 active+undersized+degraded, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2024-07-15T07:01:05.371 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:05 smithi089 bash[82202]: audit 2024-07-15T07:01:04.795436+0000 mon.a (mon.0) 295 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:01:05.371 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:05 smithi089 bash[82202]: audit 2024-07-15T07:01:04.797264+0000 mon.a (mon.0) 296 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:01:05.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:05 smithi134 bash[67624]: audit 2024-07-15T07:01:04.179744+0000 mon.a (mon.0) 292 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:01:05.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:05 smithi134 bash[67624]: audit 2024-07-15T07:01:04.185457+0000 mon.a (mon.0) 293 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:01:05.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:05 smithi134 bash[67624]: audit 2024-07-15T07:01:04.186684+0000 mon.a (mon.0) 294 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T07:01:05.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:05 smithi134 bash[67624]: audit 2024-07-15T07:01:04.532520+0000 mon.c (mon.2) 3 : audit [DBG] from='client.? 172.21.15.89:0/2559401453' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-15T07:01:05.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:05 smithi134 bash[67624]: cluster 2024-07-15T07:01:04.793149+0000 mgr.y (mgr.25244) 186 : cluster [DBG] pgmap v125: 73 pgs: 19 active+undersized, 7 active+undersized+degraded, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2024-07-15T07:01:05.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:05 smithi134 bash[67624]: audit 2024-07-15T07:01:04.795436+0000 mon.a (mon.0) 295 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:01:05.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:05 smithi134 bash[67624]: audit 2024-07-15T07:01:04.797264+0000 mon.a (mon.0) 296 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:01:05.900 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -e sha1=6ea7aaa9f93d34dc4d21862afe00140dfddbd706 -- bash -c 'ceph orch ps' 2024-07-15T07:01:06.802 INFO:journalctl@ceph.osd.0.smithi089.stdout:Jul 15 07:01:06 smithi089 bash[88290]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-07-15T07:01:06.803 INFO:journalctl@ceph.osd.0.smithi089.stdout:Jul 15 07:01:06 smithi089 bash[88290]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-07-15T07:01:06.803 INFO:journalctl@ceph.osd.0.smithi089.stdout:Jul 15 07:01:06 smithi089 bash[88290]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-07-15T07:01:06.803 INFO:journalctl@ceph.osd.0.smithi089.stdout:Jul 15 07:01:06 smithi089 bash[88290]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-07-15T07:01:06.803 INFO:journalctl@ceph.osd.0.smithi089.stdout:Jul 15 07:01:06 smithi089 bash[88290]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-07-15T07:01:06.803 INFO:journalctl@ceph.osd.0.smithi089.stdout:Jul 15 07:01:06 smithi089 bash[88290]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-07-15T07:01:06.803 INFO:journalctl@ceph.osd.0.smithi089.stdout:Jul 15 07:01:06 smithi089 bash[88290]: --> ceph-volume raw activate successful for osd ID: 0 2024-07-15T07:01:07.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:06 smithi089 bash[79810]: cluster 2024-07-15T07:01:06.794387+0000 mgr.y (mgr.25244) 187 : cluster [DBG] pgmap v126: 73 pgs: 19 active+undersized, 7 active+undersized+degraded, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2024-07-15T07:01:07.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:06 smithi089 bash[82202]: cluster 2024-07-15T07:01:06.794387+0000 mgr.y (mgr.25244) 187 : cluster [DBG] pgmap v126: 73 pgs: 19 active+undersized, 7 active+undersized+degraded, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2024-07-15T07:01:07.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:06 smithi134 bash[67624]: cluster 2024-07-15T07:01:06.794387+0000 mgr.y (mgr.25244) 187 : cluster [DBG] pgmap v126: 73 pgs: 19 active+undersized, 7 active+undersized+degraded, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2024-07-15T07:01:09.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:08 smithi089 bash[79810]: cluster 2024-07-15T07:01:08.795086+0000 mgr.y (mgr.25244) 188 : cluster [DBG] pgmap v127: 73 pgs: 19 active+undersized, 7 active+undersized+degraded, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2024-07-15T07:01:09.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:08 smithi089 bash[82202]: cluster 2024-07-15T07:01:08.795086+0000 mgr.y (mgr.25244) 188 : cluster [DBG] pgmap v127: 73 pgs: 19 active+undersized, 7 active+undersized+degraded, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2024-07-15T07:01:09.120 INFO:journalctl@ceph.osd.0.smithi089.stdout:Jul 15 07:01:08 smithi089 bash[88569]: debug 2024-07-15T07:01:08.746+0000 7f165c512640 1 -- 172.21.15.89:0/720924938 <== mon.0 v2:172.21.15.89:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55dc2c69b1e0 con 0x55dc2c65bc00 2024-07-15T07:01:09.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:08 smithi134 bash[67624]: cluster 2024-07-15T07:01:08.795086+0000 mgr.y (mgr.25244) 188 : cluster [DBG] pgmap v127: 73 pgs: 19 active+undersized, 7 active+undersized+degraded, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2024-07-15T07:01:09.619 INFO:journalctl@ceph.osd.0.smithi089.stdout:Jul 15 07:01:09 smithi089 bash[88569]: debug 2024-07-15T07:01:09.302+0000 7f165ed78640 -1 Falling back to public interface 2024-07-15T07:01:10.912 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T07:01:11.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:10 smithi134 bash[67624]: cluster 2024-07-15T07:01:10.795926+0000 mgr.y (mgr.25244) 189 : cluster [DBG] pgmap v128: 73 pgs: 19 active+undersized, 7 active+undersized+degraded, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2024-07-15T07:01:11.369 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:10 smithi089 bash[79810]: cluster 2024-07-15T07:01:10.795926+0000 mgr.y (mgr.25244) 189 : cluster [DBG] pgmap v128: 73 pgs: 19 active+undersized, 7 active+undersized+degraded, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2024-07-15T07:01:11.369 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:10 smithi089 bash[82202]: cluster 2024-07-15T07:01:10.795926+0000 mgr.y (mgr.25244) 189 : cluster [DBG] pgmap v128: 73 pgs: 19 active+undersized, 7 active+undersized+degraded, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2024-07-15T07:01:12.550 INFO:journalctl@ceph.osd.0.smithi089.stdout:Jul 15 07:01:12 smithi089 bash[88569]: debug 2024-07-15T07:01:12.325+0000 7f165ed78640 -1 osd.0 79 log_to_monitors true 2024-07-15T07:01:12.805 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:01:12 smithi134 bash[60729]: ts=2024-07-15T07:01:12.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T07:01:12.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:12 smithi134 bash[67624]: audit 2024-07-15T07:01:11.555542+0000 mon.a (mon.0) 297 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:01:12.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:12 smithi134 bash[67624]: audit 2024-07-15T07:01:11.560552+0000 mon.a (mon.0) 298 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:01:12.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:12 smithi134 bash[67624]: audit 2024-07-15T07:01:12.222376+0000 mon.a (mon.0) 299 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:01:12.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:12 smithi134 bash[67624]: audit 2024-07-15T07:01:12.230276+0000 mon.a (mon.0) 300 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:01:12.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:12 smithi134 bash[67624]: audit 2024-07-15T07:01:12.333082+0000 mon.a (mon.0) 301 : audit [INF] from='osd.0 [v2:172.21.15.89:6802/1073465095,v1:172.21.15.89:6803/1073465095]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-07-15T07:01:12.869 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:12 smithi089 bash[82202]: audit 2024-07-15T07:01:11.555542+0000 mon.a (mon.0) 297 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:01:12.870 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:12 smithi089 bash[82202]: audit 2024-07-15T07:01:11.560552+0000 mon.a (mon.0) 298 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:01:12.870 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:12 smithi089 bash[82202]: audit 2024-07-15T07:01:12.222376+0000 mon.a (mon.0) 299 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:01:12.870 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:12 smithi089 bash[82202]: audit 2024-07-15T07:01:12.230276+0000 mon.a (mon.0) 300 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:01:12.870 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:12 smithi089 bash[82202]: audit 2024-07-15T07:01:12.333082+0000 mon.a (mon.0) 301 : audit [INF] from='osd.0 [v2:172.21.15.89:6802/1073465095,v1:172.21.15.89:6803/1073465095]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-07-15T07:01:12.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:12 smithi089 bash[79810]: audit 2024-07-15T07:01:11.555542+0000 mon.a (mon.0) 297 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:01:12.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:12 smithi089 bash[79810]: audit 2024-07-15T07:01:11.560552+0000 mon.a (mon.0) 298 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:01:12.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:12 smithi089 bash[79810]: audit 2024-07-15T07:01:12.222376+0000 mon.a (mon.0) 299 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:01:12.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:12 smithi089 bash[79810]: audit 2024-07-15T07:01:12.230276+0000 mon.a (mon.0) 300 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:01:12.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:12 smithi089 bash[79810]: audit 2024-07-15T07:01:12.333082+0000 mon.a (mon.0) 301 : audit [INF] from='osd.0 [v2:172.21.15.89:6802/1073465095,v1:172.21.15.89:6803/1073465095]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-07-15T07:01:14.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:13 smithi134 bash[67624]: cluster 2024-07-15T07:01:12.796625+0000 mgr.y (mgr.25244) 190 : cluster [DBG] pgmap v129: 73 pgs: 19 active+undersized, 7 active+undersized+degraded, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2024-07-15T07:01:14.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:13 smithi134 bash[67624]: audit 2024-07-15T07:01:13.285821+0000 mon.a (mon.0) 302 : audit [INF] from='osd.0 [v2:172.21.15.89:6802/1073465095,v1:172.21.15.89:6803/1073465095]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-07-15T07:01:14.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:13 smithi134 bash[67624]: cluster 2024-07-15T07:01:13.285963+0000 mon.a (mon.0) 303 : cluster [DBG] osdmap e82: 8 total, 7 up, 8 in 2024-07-15T07:01:14.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:13 smithi134 bash[67624]: audit 2024-07-15T07:01:13.286758+0000 mon.a (mon.0) 304 : audit [INF] from='osd.0 [v2:172.21.15.89:6802/1073465095,v1:172.21.15.89:6803/1073465095]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2024-07-15T07:01:14.119 INFO:journalctl@ceph.osd.0.smithi089.stdout:Jul 15 07:01:13 smithi089 bash[88569]: debug 2024-07-15T07:01:13.765+0000 7f1652b1a640 -1 osd.0 79 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-07-15T07:01:14.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:13 smithi089 bash[79810]: cluster 2024-07-15T07:01:12.796625+0000 mgr.y (mgr.25244) 190 : cluster [DBG] pgmap v129: 73 pgs: 19 active+undersized, 7 active+undersized+degraded, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2024-07-15T07:01:14.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:13 smithi089 bash[79810]: audit 2024-07-15T07:01:13.285821+0000 mon.a (mon.0) 302 : audit [INF] from='osd.0 [v2:172.21.15.89:6802/1073465095,v1:172.21.15.89:6803/1073465095]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-07-15T07:01:14.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:13 smithi089 bash[79810]: cluster 2024-07-15T07:01:13.285963+0000 mon.a (mon.0) 303 : cluster [DBG] osdmap e82: 8 total, 7 up, 8 in 2024-07-15T07:01:14.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:13 smithi089 bash[79810]: audit 2024-07-15T07:01:13.286758+0000 mon.a (mon.0) 304 : audit [INF] from='osd.0 [v2:172.21.15.89:6802/1073465095,v1:172.21.15.89:6803/1073465095]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2024-07-15T07:01:14.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:13 smithi089 bash[82202]: cluster 2024-07-15T07:01:12.796625+0000 mgr.y (mgr.25244) 190 : cluster [DBG] pgmap v129: 73 pgs: 19 active+undersized, 7 active+undersized+degraded, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2024-07-15T07:01:14.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:13 smithi089 bash[82202]: audit 2024-07-15T07:01:13.285821+0000 mon.a (mon.0) 302 : audit [INF] from='osd.0 [v2:172.21.15.89:6802/1073465095,v1:172.21.15.89:6803/1073465095]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-07-15T07:01:14.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:13 smithi089 bash[82202]: cluster 2024-07-15T07:01:13.285963+0000 mon.a (mon.0) 303 : cluster [DBG] osdmap e82: 8 total, 7 up, 8 in 2024-07-15T07:01:14.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:13 smithi089 bash[82202]: audit 2024-07-15T07:01:13.286758+0000 mon.a (mon.0) 304 : audit [INF] from='osd.0 [v2:172.21.15.89:6802/1073465095,v1:172.21.15.89:6803/1073465095]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2024-07-15T07:01:14.619 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 07:01:14 smithi089 bash[72894]: ::ffff:172.21.15.134 - - [15/Jul/2024:07:01:14] "GET /metrics HTTP/1.1" 200 33993 "" "Prometheus/2.43.0" 2024-07-15T07:01:14.664 INFO:teuthology.orchestra.run.smithi089.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-07-15T07:01:14.664 INFO:teuthology.orchestra.run.smithi089.stdout:alertmanager.a smithi089 *:9093,9094 running (5m) 3s ago 13m 13.8M - 0.25.0 c8568f914cd2 c476af8941dd 2024-07-15T07:01:14.664 INFO:teuthology.orchestra.run.smithi089.stdout:grafana.a smithi134 *:3000 running (5m) 27s ago 12m 76.0M - 9.4.7 954c08fa6188 ac957811e4af 2024-07-15T07:01:14.664 INFO:teuthology.orchestra.run.smithi089.stdout:mds.foo.smithi089.cczuoz smithi089 running (10m) 3s ago 10m 14.3M - 17.2.7-1102-g9a202024 280b03ed6869 a1f028849336 2024-07-15T07:01:14.664 INFO:teuthology.orchestra.run.smithi089.stdout:mds.foo.smithi134.wugtzh smithi134 running (10m) 27s ago 10m 16.2M - 17.2.7-1102-g9a202024 280b03ed6869 fb5bbeafa32b 2024-07-15T07:01:14.664 INFO:teuthology.orchestra.run.smithi089.stdout:mgr.x smithi134 *:8443,9283,8765 running (4m) 27s ago 21m 424M - 18.2.2-1609-g6ea7aaa9 0add76435ed9 842a5eaedc06 2024-07-15T07:01:14.664 INFO:teuthology.orchestra.run.smithi089.stdout:mgr.y smithi089 *:8443,9283,8765 running (5m) 3s ago 23m 479M - 18.2.2-1609-g6ea7aaa9 0add76435ed9 20c56e260e99 2024-07-15T07:01:14.665 INFO:teuthology.orchestra.run.smithi089.stdout:mon.a smithi089 running (4m) 3s ago 24m 43.2M 2048M 18.2.2-1609-g6ea7aaa9 0add76435ed9 b2d2728b9d78 2024-07-15T07:01:14.665 INFO:teuthology.orchestra.run.smithi089.stdout:mon.b smithi134 running (119s) 27s ago 21m 32.8M 2048M 18.2.2-1609-g6ea7aaa9 0add76435ed9 f157e925047f 2024-07-15T07:01:14.665 INFO:teuthology.orchestra.run.smithi089.stdout:mon.c smithi089 running (2m) 3s ago 21m 36.9M 2048M 18.2.2-1609-g6ea7aaa9 0add76435ed9 60c08b0bcea8 2024-07-15T07:01:14.665 INFO:teuthology.orchestra.run.smithi089.stdout:node-exporter.a smithi089 *:9100 running (5m) 3s ago 13m 7980k - 1.5.0 0da6a335fe13 5da5d79c182f 2024-07-15T07:01:14.665 INFO:teuthology.orchestra.run.smithi089.stdout:node-exporter.b smithi134 *:9100 running (5m) 27s ago 13m 8464k - 1.5.0 0da6a335fe13 9ace5df7006d 2024-07-15T07:01:14.665 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0 smithi089 running (7s) 3s ago 20m - 2524M 18.2.2-1609-g6ea7aaa9 0add76435ed9 62162797039a 2024-07-15T07:01:14.665 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1 smithi089 running (19m) 3s ago 19m 50.6M 2524M 17.2.7-1102-g9a202024 280b03ed6869 b4ff988d2ed5 2024-07-15T07:01:14.665 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2 smithi089 running (19m) 3s ago 19m 56.7M 2524M 17.2.7-1102-g9a202024 280b03ed6869 428fbfd265d1 2024-07-15T07:01:14.665 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3 smithi089 running (18m) 3s ago 18m 46.7M 2524M 17.2.7-1102-g9a202024 280b03ed6869 4f43743cd007 2024-07-15T07:01:14.665 INFO:teuthology.orchestra.run.smithi089.stdout:osd.4 smithi134 running (17m) 27s ago 17m 51.9M 2524M 17.2.7-1102-g9a202024 280b03ed6869 2ae2fa51d983 2024-07-15T07:01:14.665 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5 smithi134 running (16m) 27s ago 16m 54.9M 2524M 17.2.7-1102-g9a202024 280b03ed6869 61be14c58fc3 2024-07-15T07:01:14.665 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6 smithi134 running (15m) 27s ago 15m 54.1M 2524M 17.2.7-1102-g9a202024 280b03ed6869 f92c7dd84666 2024-07-15T07:01:14.665 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7 smithi134 running (14m) 27s ago 14m 54.6M 2524M 17.2.7-1102-g9a202024 280b03ed6869 e44d11319ff6 2024-07-15T07:01:14.665 INFO:teuthology.orchestra.run.smithi089.stdout:prometheus.a smithi134 *:9095 running (4m) 27s ago 13m 56.5M - 2.43.0 a07b618ecd1d 8873e527bacf 2024-07-15T07:01:14.934 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:14 smithi089 bash[82202]: audit 2024-07-15T07:01:13.761286+0000 mon.a (mon.0) 305 : audit [INF] from='osd.0 [v2:172.21.15.89:6802/1073465095,v1:172.21.15.89:6803/1073465095]' entity='osd.0' 2024-07-15T07:01:14.934 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:14 smithi089 bash[79810]: debug 2024-07-15T07:01:14.753+0000 7fb563a0a640 -1 log_channel(cluster) log [ERR] : Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-07-15T07:01:14.935 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:14 smithi089 bash[79810]: audit 2024-07-15T07:01:13.761286+0000 mon.a (mon.0) 305 : audit [INF] from='osd.0 [v2:172.21.15.89:6802/1073465095,v1:172.21.15.89:6803/1073465095]' entity='osd.0' 2024-07-15T07:01:15.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:14 smithi134 bash[67624]: audit 2024-07-15T07:01:13.761286+0000 mon.a (mon.0) 305 : audit [INF] from='osd.0 [v2:172.21.15.89:6802/1073465095,v1:172.21.15.89:6803/1073465095]' entity='osd.0' 2024-07-15T07:01:15.538 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -e sha1=6ea7aaa9f93d34dc4d21862afe00140dfddbd706 -- bash -c 'ceph orch upgrade pause' 2024-07-15T07:01:15.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:15 smithi089 bash[79810]: audit 2024-07-15T07:01:14.645636+0000 mgr.y (mgr.25244) 191 : audit [DBG] from='client.34203 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T07:01:15.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:15 smithi089 bash[79810]: cluster 2024-07-15T07:01:14.756767+0000 mon.a (mon.0) 306 : cluster [ERR] Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-07-15T07:01:15.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:15 smithi089 bash[79810]: cluster 2024-07-15T07:01:14.756786+0000 mon.a (mon.0) 307 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-07-15T07:01:15.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:15 smithi089 bash[79810]: cluster 2024-07-15T07:01:14.764261+0000 mon.a (mon.0) 308 : cluster [INF] osd.0 [v2:172.21.15.89:6802/1073465095,v1:172.21.15.89:6803/1073465095] boot 2024-07-15T07:01:15.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:15 smithi089 bash[79810]: cluster 2024-07-15T07:01:14.764302+0000 mon.a (mon.0) 309 : cluster [DBG] osdmap e83: 8 total, 8 up, 8 in 2024-07-15T07:01:15.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:15 smithi089 bash[79810]: audit 2024-07-15T07:01:14.764887+0000 mon.a (mon.0) 310 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-07-15T07:01:15.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:15 smithi089 bash[79810]: cluster 2024-07-15T07:01:14.797063+0000 mgr.y (mgr.25244) 192 : cluster [DBG] pgmap v132: 73 pgs: 19 active+undersized, 7 active+undersized+degraded, 47 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2024-07-15T07:01:15.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:15 smithi089 bash[79810]: cluster 2024-07-15T07:01:15.766165+0000 mon.a (mon.0) 311 : cluster [DBG] osdmap e84: 8 total, 8 up, 8 in 2024-07-15T07:01:15.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:15 smithi089 bash[82202]: audit 2024-07-15T07:01:14.645636+0000 mgr.y (mgr.25244) 191 : audit [DBG] from='client.34203 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T07:01:15.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:15 smithi089 bash[82202]: cluster 2024-07-15T07:01:14.756767+0000 mon.a (mon.0) 306 : cluster [ERR] Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-07-15T07:01:15.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:15 smithi089 bash[82202]: cluster 2024-07-15T07:01:14.756786+0000 mon.a (mon.0) 307 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-07-15T07:01:15.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:15 smithi089 bash[82202]: cluster 2024-07-15T07:01:14.764261+0000 mon.a (mon.0) 308 : cluster [INF] osd.0 [v2:172.21.15.89:6802/1073465095,v1:172.21.15.89:6803/1073465095] boot 2024-07-15T07:01:15.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:15 smithi089 bash[82202]: cluster 2024-07-15T07:01:14.764302+0000 mon.a (mon.0) 309 : cluster [DBG] osdmap e83: 8 total, 8 up, 8 in 2024-07-15T07:01:15.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:15 smithi089 bash[82202]: audit 2024-07-15T07:01:14.764887+0000 mon.a (mon.0) 310 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-07-15T07:01:15.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:15 smithi089 bash[82202]: cluster 2024-07-15T07:01:14.797063+0000 mgr.y (mgr.25244) 192 : cluster [DBG] pgmap v132: 73 pgs: 19 active+undersized, 7 active+undersized+degraded, 47 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2024-07-15T07:01:15.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:15 smithi089 bash[82202]: cluster 2024-07-15T07:01:15.766165+0000 mon.a (mon.0) 311 : cluster [DBG] osdmap e84: 8 total, 8 up, 8 in 2024-07-15T07:01:16.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:15 smithi134 bash[67624]: audit 2024-07-15T07:01:14.645636+0000 mgr.y (mgr.25244) 191 : audit [DBG] from='client.34203 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T07:01:16.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:15 smithi134 bash[67624]: cluster 2024-07-15T07:01:14.756767+0000 mon.a (mon.0) 306 : cluster [ERR] Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-07-15T07:01:16.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:15 smithi134 bash[67624]: cluster 2024-07-15T07:01:14.756786+0000 mon.a (mon.0) 307 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-07-15T07:01:16.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:15 smithi134 bash[67624]: cluster 2024-07-15T07:01:14.764261+0000 mon.a (mon.0) 308 : cluster [INF] osd.0 [v2:172.21.15.89:6802/1073465095,v1:172.21.15.89:6803/1073465095] boot 2024-07-15T07:01:16.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:15 smithi134 bash[67624]: cluster 2024-07-15T07:01:14.764302+0000 mon.a (mon.0) 309 : cluster [DBG] osdmap e83: 8 total, 8 up, 8 in 2024-07-15T07:01:16.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:15 smithi134 bash[67624]: audit 2024-07-15T07:01:14.764887+0000 mon.a (mon.0) 310 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-07-15T07:01:16.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:15 smithi134 bash[67624]: cluster 2024-07-15T07:01:14.797063+0000 mgr.y (mgr.25244) 192 : cluster [DBG] pgmap v132: 73 pgs: 19 active+undersized, 7 active+undersized+degraded, 47 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2024-07-15T07:01:16.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:15 smithi134 bash[67624]: cluster 2024-07-15T07:01:15.766165+0000 mon.a (mon.0) 311 : cluster [DBG] osdmap e84: 8 total, 8 up, 8 in 2024-07-15T07:01:17.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:16 smithi134 bash[67624]: cluster 2024-07-15T07:01:16.798108+0000 mgr.y (mgr.25244) 193 : cluster [DBG] pgmap v134: 73 pgs: 5 activating, 10 peering, 9 active+undersized, 2 active+undersized+degraded, 47 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2024-07-15T07:01:17.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:16 smithi134 bash[67624]: cluster 2024-07-15T07:01:16.849606+0000 mon.a (mon.0) 312 : cluster [WRN] Health check update: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded (PG_DEGRADED) 2024-07-15T07:01:17.369 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:16 smithi089 bash[82202]: cluster 2024-07-15T07:01:16.798108+0000 mgr.y (mgr.25244) 193 : cluster [DBG] pgmap v134: 73 pgs: 5 activating, 10 peering, 9 active+undersized, 2 active+undersized+degraded, 47 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2024-07-15T07:01:17.369 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:16 smithi089 bash[82202]: cluster 2024-07-15T07:01:16.849606+0000 mon.a (mon.0) 312 : cluster [WRN] Health check update: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded (PG_DEGRADED) 2024-07-15T07:01:17.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:16 smithi089 bash[79810]: cluster 2024-07-15T07:01:16.798108+0000 mgr.y (mgr.25244) 193 : cluster [DBG] pgmap v134: 73 pgs: 5 activating, 10 peering, 9 active+undersized, 2 active+undersized+degraded, 47 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2024-07-15T07:01:17.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:16 smithi089 bash[79810]: cluster 2024-07-15T07:01:16.849606+0000 mon.a (mon.0) 312 : cluster [WRN] Health check update: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded (PG_DEGRADED) 2024-07-15T07:01:19.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:18 smithi134 bash[67624]: cluster 2024-07-15T07:01:18.798839+0000 mgr.y (mgr.25244) 194 : cluster [DBG] pgmap v135: 73 pgs: 5 activating, 10 peering, 58 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:01:19.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:18 smithi134 bash[67624]: cluster 2024-07-15T07:01:18.850360+0000 mon.a (mon.0) 313 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded) 2024-07-15T07:01:19.369 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:18 smithi089 bash[82202]: cluster 2024-07-15T07:01:18.798839+0000 mgr.y (mgr.25244) 194 : cluster [DBG] pgmap v135: 73 pgs: 5 activating, 10 peering, 58 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:01:19.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:18 smithi089 bash[82202]: cluster 2024-07-15T07:01:18.850360+0000 mon.a (mon.0) 313 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded) 2024-07-15T07:01:19.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:18 smithi089 bash[79810]: cluster 2024-07-15T07:01:18.798839+0000 mgr.y (mgr.25244) 194 : cluster [DBG] pgmap v135: 73 pgs: 5 activating, 10 peering, 58 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:01:19.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:18 smithi089 bash[79810]: cluster 2024-07-15T07:01:18.850360+0000 mon.a (mon.0) 313 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded) 2024-07-15T07:01:20.869 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:20 smithi089 bash[79810]: audit 2024-07-15T07:01:19.793089+0000 mon.a (mon.0) 314 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:01:20.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:20 smithi089 bash[79810]: audit 2024-07-15T07:01:19.793947+0000 mon.a (mon.0) 315 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:01:20.870 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:20 smithi089 bash[82202]: audit 2024-07-15T07:01:19.793089+0000 mon.a (mon.0) 314 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:01:20.870 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:20 smithi089 bash[82202]: audit 2024-07-15T07:01:19.793947+0000 mon.a (mon.0) 315 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:01:20.893 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T07:01:21.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:20 smithi134 bash[67624]: audit 2024-07-15T07:01:19.793089+0000 mon.a (mon.0) 314 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:01:21.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:20 smithi134 bash[67624]: audit 2024-07-15T07:01:19.793947+0000 mon.a (mon.0) 315 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:01:22.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:21 smithi134 bash[67624]: cluster 2024-07-15T07:01:20.799901+0000 mgr.y (mgr.25244) 195 : cluster [DBG] pgmap v136: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:01:22.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:21 smithi089 bash[79810]: cluster 2024-07-15T07:01:20.799901+0000 mgr.y (mgr.25244) 195 : cluster [DBG] pgmap v136: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:01:22.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:21 smithi089 bash[82202]: cluster 2024-07-15T07:01:20.799901+0000 mgr.y (mgr.25244) 195 : cluster [DBG] pgmap v136: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:01:22.804 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:01:22 smithi134 bash[60729]: ts=2024-07-15T07:01:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T07:01:23.115 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:22 smithi089 bash[79810]: cluster 2024-07-15T07:01:22.800486+0000 mgr.y (mgr.25244) 196 : cluster [DBG] pgmap v137: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:01:23.116 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:22 smithi089 bash[82202]: cluster 2024-07-15T07:01:22.800486+0000 mgr.y (mgr.25244) 196 : cluster [DBG] pgmap v137: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:01:23.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:22 smithi134 bash[67624]: cluster 2024-07-15T07:01:22.800486+0000 mgr.y (mgr.25244) 196 : cluster [DBG] pgmap v137: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:01:24.619 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 07:01:24 smithi089 bash[72894]: ::ffff:172.21.15.134 - - [15/Jul/2024:07:01:24] "GET /metrics HTTP/1.1" 200 33982 "" "Prometheus/2.43.0" 2024-07-15T07:01:25.821 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:25 smithi089 bash[79810]: audit 2024-07-15T07:01:24.567052+0000 mon.a (mon.0) 316 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:01:25.821 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:25 smithi089 bash[79810]: audit 2024-07-15T07:01:24.576888+0000 mon.a (mon.0) 317 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:01:25.822 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:25 smithi089 bash[79810]: audit 2024-07-15T07:01:24.578894+0000 mon.a (mon.0) 318 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T07:01:25.822 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:25 smithi089 bash[79810]: audit 2024-07-15T07:01:24.580720+0000 mon.a (mon.0) 319 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T07:01:25.822 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:25 smithi089 bash[79810]: audit 2024-07-15T07:01:24.590829+0000 mon.a (mon.0) 320 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:01:25.822 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:25 smithi089 bash[79810]: audit 2024-07-15T07:01:24.698351+0000 mon.a (mon.0) 321 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T07:01:25.822 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:25 smithi089 bash[79810]: audit 2024-07-15T07:01:24.703220+0000 mon.a (mon.0) 322 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-15T07:01:25.822 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:25 smithi089 bash[79810]: audit 2024-07-15T07:01:24.705688+0000 mon.a (mon.0) 323 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-15T07:01:25.822 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:25 smithi089 bash[79810]: audit 2024-07-15T07:01:24.707709+0000 mon.a (mon.0) 324 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-15T07:01:25.822 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:25 smithi089 bash[79810]: audit 2024-07-15T07:01:24.710043+0000 mon.a (mon.0) 325 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-07-15T07:01:25.822 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:25 smithi089 bash[79810]: audit 2024-07-15T07:01:24.710425+0000 mgr.y (mgr.25244) 197 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-07-15T07:01:25.822 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:25 smithi089 bash[79810]: cephadm 2024-07-15T07:01:24.712282+0000 mgr.y (mgr.25244) 198 : cephadm [INF] Upgrade: osd.1 is safe to restart 2024-07-15T07:01:25.822 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:25 smithi089 bash[79810]: cluster 2024-07-15T07:01:24.801235+0000 mgr.y (mgr.25244) 199 : cluster [DBG] pgmap v138: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:01:25.823 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:25 smithi089 bash[82202]: audit 2024-07-15T07:01:24.567052+0000 mon.a (mon.0) 316 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:01:25.823 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:25 smithi089 bash[82202]: audit 2024-07-15T07:01:24.576888+0000 mon.a (mon.0) 317 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:01:25.823 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:25 smithi089 bash[82202]: audit 2024-07-15T07:01:24.578894+0000 mon.a (mon.0) 318 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T07:01:25.823 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:25 smithi089 bash[82202]: audit 2024-07-15T07:01:24.580720+0000 mon.a (mon.0) 319 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T07:01:25.823 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:25 smithi089 bash[82202]: audit 2024-07-15T07:01:24.590829+0000 mon.a (mon.0) 320 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:01:25.823 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:25 smithi089 bash[82202]: audit 2024-07-15T07:01:24.698351+0000 mon.a (mon.0) 321 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T07:01:25.823 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:25 smithi089 bash[82202]: audit 2024-07-15T07:01:24.703220+0000 mon.a (mon.0) 322 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-15T07:01:25.823 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:25 smithi089 bash[82202]: audit 2024-07-15T07:01:24.705688+0000 mon.a (mon.0) 323 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-15T07:01:25.823 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:25 smithi089 bash[82202]: audit 2024-07-15T07:01:24.707709+0000 mon.a (mon.0) 324 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-15T07:01:25.823 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:25 smithi089 bash[82202]: audit 2024-07-15T07:01:24.710043+0000 mon.a (mon.0) 325 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-07-15T07:01:25.823 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:25 smithi089 bash[82202]: audit 2024-07-15T07:01:24.710425+0000 mgr.y (mgr.25244) 197 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-07-15T07:01:25.823 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:25 smithi089 bash[82202]: cephadm 2024-07-15T07:01:24.712282+0000 mgr.y (mgr.25244) 198 : cephadm [INF] Upgrade: osd.1 is safe to restart 2024-07-15T07:01:25.823 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:25 smithi089 bash[82202]: cluster 2024-07-15T07:01:24.801235+0000 mgr.y (mgr.25244) 199 : cluster [DBG] pgmap v138: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:01:26.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:25 smithi134 bash[67624]: audit 2024-07-15T07:01:24.567052+0000 mon.a (mon.0) 316 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:01:26.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:25 smithi134 bash[67624]: audit 2024-07-15T07:01:24.576888+0000 mon.a (mon.0) 317 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:01:26.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:25 smithi134 bash[67624]: audit 2024-07-15T07:01:24.578894+0000 mon.a (mon.0) 318 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T07:01:26.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:25 smithi134 bash[67624]: audit 2024-07-15T07:01:24.580720+0000 mon.a (mon.0) 319 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T07:01:26.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:25 smithi134 bash[67624]: audit 2024-07-15T07:01:24.590829+0000 mon.a (mon.0) 320 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:01:26.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:25 smithi134 bash[67624]: audit 2024-07-15T07:01:24.698351+0000 mon.a (mon.0) 321 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T07:01:26.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:25 smithi134 bash[67624]: audit 2024-07-15T07:01:24.703220+0000 mon.a (mon.0) 322 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-15T07:01:26.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:25 smithi134 bash[67624]: audit 2024-07-15T07:01:24.705688+0000 mon.a (mon.0) 323 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-15T07:01:26.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:25 smithi134 bash[67624]: audit 2024-07-15T07:01:24.707709+0000 mon.a (mon.0) 324 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-15T07:01:26.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:25 smithi134 bash[67624]: audit 2024-07-15T07:01:24.710043+0000 mon.a (mon.0) 325 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-07-15T07:01:26.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:25 smithi134 bash[67624]: audit 2024-07-15T07:01:24.710425+0000 mgr.y (mgr.25244) 197 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-07-15T07:01:26.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:25 smithi134 bash[67624]: cephadm 2024-07-15T07:01:24.712282+0000 mgr.y (mgr.25244) 198 : cephadm [INF] Upgrade: osd.1 is safe to restart 2024-07-15T07:01:26.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:25 smithi134 bash[67624]: cluster 2024-07-15T07:01:24.801235+0000 mgr.y (mgr.25244) 199 : cluster [DBG] pgmap v138: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:01:26.196 INFO:teuthology.orchestra.run.smithi089.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:badc66b92d2a2cfbbce7f293b5accdc50f486d2b2d966876e46fe28d05f464f1 2024-07-15T07:01:27.290 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -e sha1=6ea7aaa9f93d34dc4d21862afe00140dfddbd706 -- bash -c 'sleep 60' 2024-07-15T07:01:27.369 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:27 smithi089 bash[82202]: audit 2024-07-15T07:01:26.194226+0000 mgr.y (mgr.25244) 200 : audit [DBG] from='client.34209 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T07:01:27.369 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:27 smithi089 bash[82202]: cephadm 2024-07-15T07:01:26.194758+0000 mgr.y (mgr.25244) 201 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:badc66b92d2a2cfbbce7f293b5accdc50f486d2b2d966876e46fe28d05f464f1 2024-07-15T07:01:27.369 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:27 smithi089 bash[82202]: audit 2024-07-15T07:01:26.201302+0000 mon.a (mon.0) 326 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:01:27.369 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:27 smithi089 bash[82202]: cluster 2024-07-15T07:01:26.802284+0000 mgr.y (mgr.25244) 202 : cluster [DBG] pgmap v139: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:01:27.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:27 smithi089 bash[79810]: audit 2024-07-15T07:01:26.194226+0000 mgr.y (mgr.25244) 200 : audit [DBG] from='client.34209 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T07:01:27.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:27 smithi089 bash[79810]: cephadm 2024-07-15T07:01:26.194758+0000 mgr.y (mgr.25244) 201 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:badc66b92d2a2cfbbce7f293b5accdc50f486d2b2d966876e46fe28d05f464f1 2024-07-15T07:01:27.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:27 smithi089 bash[79810]: audit 2024-07-15T07:01:26.201302+0000 mon.a (mon.0) 326 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:01:27.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:27 smithi089 bash[79810]: cluster 2024-07-15T07:01:26.802284+0000 mgr.y (mgr.25244) 202 : cluster [DBG] pgmap v139: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:01:27.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:27 smithi134 bash[67624]: audit 2024-07-15T07:01:26.194226+0000 mgr.y (mgr.25244) 200 : audit [DBG] from='client.34209 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T07:01:27.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:27 smithi134 bash[67624]: cephadm 2024-07-15T07:01:26.194758+0000 mgr.y (mgr.25244) 201 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:badc66b92d2a2cfbbce7f293b5accdc50f486d2b2d966876e46fe28d05f464f1 2024-07-15T07:01:27.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:27 smithi134 bash[67624]: audit 2024-07-15T07:01:26.201302+0000 mon.a (mon.0) 326 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:01:27.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:27 smithi134 bash[67624]: cluster 2024-07-15T07:01:26.802284+0000 mgr.y (mgr.25244) 202 : cluster [DBG] pgmap v139: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:01:29.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:29 smithi134 bash[67624]: cephadm 2024-07-15T07:01:28.012982+0000 mgr.y (mgr.25244) 203 : cephadm [INF] Upgrade: Updating osd.1 2024-07-15T07:01:29.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:29 smithi134 bash[67624]: audit 2024-07-15T07:01:28.022282+0000 mon.a (mon.0) 327 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:01:29.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:29 smithi134 bash[67624]: audit 2024-07-15T07:01:28.024189+0000 mon.a (mon.0) 328 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-07-15T07:01:29.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:29 smithi134 bash[67624]: audit 2024-07-15T07:01:28.025706+0000 mon.a (mon.0) 329 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T07:01:29.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:29 smithi134 bash[67624]: cephadm 2024-07-15T07:01:28.029910+0000 mgr.y (mgr.25244) 204 : cephadm [INF] Deploying daemon osd.1 on smithi089 2024-07-15T07:01:29.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:29 smithi134 bash[67624]: cluster 2024-07-15T07:01:28.802990+0000 mgr.y (mgr.25244) 205 : cluster [DBG] pgmap v140: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:01:29.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:29 smithi089 bash[79810]: cephadm 2024-07-15T07:01:28.012982+0000 mgr.y (mgr.25244) 203 : cephadm [INF] Upgrade: Updating osd.1 2024-07-15T07:01:29.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:29 smithi089 bash[79810]: audit 2024-07-15T07:01:28.022282+0000 mon.a (mon.0) 327 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:01:29.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:29 smithi089 bash[79810]: audit 2024-07-15T07:01:28.024189+0000 mon.a (mon.0) 328 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-07-15T07:01:29.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:29 smithi089 bash[79810]: audit 2024-07-15T07:01:28.025706+0000 mon.a (mon.0) 329 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T07:01:29.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:29 smithi089 bash[79810]: cephadm 2024-07-15T07:01:28.029910+0000 mgr.y (mgr.25244) 204 : cephadm [INF] Deploying daemon osd.1 on smithi089 2024-07-15T07:01:29.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:29 smithi089 bash[79810]: cluster 2024-07-15T07:01:28.802990+0000 mgr.y (mgr.25244) 205 : cluster [DBG] pgmap v140: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:01:29.371 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:29 smithi089 bash[82202]: cephadm 2024-07-15T07:01:28.012982+0000 mgr.y (mgr.25244) 203 : cephadm [INF] Upgrade: Updating osd.1 2024-07-15T07:01:29.371 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:29 smithi089 bash[82202]: audit 2024-07-15T07:01:28.022282+0000 mon.a (mon.0) 327 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:01:29.371 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:29 smithi089 bash[82202]: audit 2024-07-15T07:01:28.024189+0000 mon.a (mon.0) 328 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-07-15T07:01:29.372 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:29 smithi089 bash[82202]: audit 2024-07-15T07:01:28.025706+0000 mon.a (mon.0) 329 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T07:01:29.372 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:29 smithi089 bash[82202]: cephadm 2024-07-15T07:01:28.029910+0000 mgr.y (mgr.25244) 204 : cephadm [INF] Deploying daemon osd.1 on smithi089 2024-07-15T07:01:29.372 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:29 smithi089 bash[82202]: cluster 2024-07-15T07:01:28.802990+0000 mgr.y (mgr.25244) 205 : cluster [DBG] pgmap v140: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:01:31.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:30 smithi134 bash[67624]: audit 2024-07-15T07:01:29.870027+0000 mon.a (mon.0) 330 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:01:31.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:30 smithi134 bash[67624]: cluster 2024-07-15T07:01:30.804017+0000 mgr.y (mgr.25244) 206 : cluster [DBG] pgmap v141: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:01:31.369 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:30 smithi089 bash[79810]: audit 2024-07-15T07:01:29.870027+0000 mon.a (mon.0) 330 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:01:31.369 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:30 smithi089 bash[79810]: cluster 2024-07-15T07:01:30.804017+0000 mgr.y (mgr.25244) 206 : cluster [DBG] pgmap v141: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:01:31.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:30 smithi089 bash[82202]: audit 2024-07-15T07:01:29.870027+0000 mon.a (mon.0) 330 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:01:31.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:30 smithi089 bash[82202]: cluster 2024-07-15T07:01:30.804017+0000 mgr.y (mgr.25244) 206 : cluster [DBG] pgmap v141: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:01:32.046 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T07:01:32.804 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:01:32 smithi134 bash[60729]: ts=2024-07-15T07:01:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T07:01:33.232 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:32 smithi089 bash[79810]: cluster 2024-07-15T07:01:32.804711+0000 mgr.y (mgr.25244) 207 : cluster [DBG] pgmap v142: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:01:33.232 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:32 smithi089 bash[82202]: cluster 2024-07-15T07:01:32.804711+0000 mgr.y (mgr.25244) 207 : cluster [DBG] pgmap v142: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:01:33.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:32 smithi134 bash[67624]: cluster 2024-07-15T07:01:32.804711+0000 mgr.y (mgr.25244) 207 : cluster [DBG] pgmap v142: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:01:33.521 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:33 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:01:33.521 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 07:01:33 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:01:33.521 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:33 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:01:33.522 INFO:journalctl@ceph.osd.0.smithi089.stdout:Jul 15 07:01:33 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:01:33.522 INFO:journalctl@ceph.osd.1.smithi089.stdout:Jul 15 07:01:33 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:01:33.522 INFO:journalctl@ceph.osd.2.smithi089.stdout:Jul 15 07:01:33 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:01:33.523 INFO:journalctl@ceph.osd.3.smithi089.stdout:Jul 15 07:01:33 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:01:33.523 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 07:01:33 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:01:33.523 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 07:01:33 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:01:33.870 INFO:journalctl@ceph.osd.1.smithi089.stdout:Jul 15 07:01:33 smithi089 systemd[1]: Stopping Ceph osd.1 for 75ba02bc-4274-11ef-bcad-c7b262605968... 2024-07-15T07:01:33.870 INFO:journalctl@ceph.osd.1.smithi089.stdout:Jul 15 07:01:33 smithi089 bash[40138]: debug 2024-07-15T07:01:33.661+0000 7f22ca02b640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-07-15T07:01:33.870 INFO:journalctl@ceph.osd.1.smithi089.stdout:Jul 15 07:01:33 smithi089 bash[40138]: debug 2024-07-15T07:01:33.661+0000 7f22ca02b640 -1 osd.1 84 *** Got signal Terminated *** 2024-07-15T07:01:33.870 INFO:journalctl@ceph.osd.1.smithi089.stdout:Jul 15 07:01:33 smithi089 bash[40138]: debug 2024-07-15T07:01:33.661+0000 7f22ca02b640 -1 osd.1 84 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-07-15T07:01:34.159 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:33 smithi089 bash[82202]: cluster 2024-07-15T07:01:33.666985+0000 mon.a (mon.0) 331 : cluster [INF] osd.1 marked itself down and dead 2024-07-15T07:01:34.160 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:33 smithi089 bash[79810]: debug 2024-07-15T07:01:33.877+0000 7fb563a0a640 -1 log_channel(cluster) log [ERR] : Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-07-15T07:01:34.160 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:33 smithi089 bash[79810]: cluster 2024-07-15T07:01:33.666985+0000 mon.a (mon.0) 331 : cluster [INF] osd.1 marked itself down and dead 2024-07-15T07:01:34.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:33 smithi134 bash[67624]: cluster 2024-07-15T07:01:33.666985+0000 mon.a (mon.0) 331 : cluster [INF] osd.1 marked itself down and dead 2024-07-15T07:01:34.625 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 07:01:34 smithi089 bash[72894]: ::ffff:172.21.15.134 - - [15/Jul/2024:07:01:34] "GET /metrics HTTP/1.1" 200 33982 "" "Prometheus/2.43.0" 2024-07-15T07:01:34.976 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:34 smithi089 bash[82202]: cluster 2024-07-15T07:01:33.881858+0000 mon.a (mon.0) 332 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-07-15T07:01:34.976 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:34 smithi089 bash[82202]: cluster 2024-07-15T07:01:33.881901+0000 mon.a (mon.0) 333 : cluster [ERR] Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-07-15T07:01:34.976 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:34 smithi089 bash[82202]: cluster 2024-07-15T07:01:33.901474+0000 mon.a (mon.0) 334 : cluster [DBG] osdmap e85: 8 total, 7 up, 8 in 2024-07-15T07:01:34.976 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:34 smithi089 bash[82202]: audit 2024-07-15T07:01:34.793657+0000 mon.a (mon.0) 335 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:01:34.976 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:34 smithi089 bash[82202]: audit 2024-07-15T07:01:34.794709+0000 mon.a (mon.0) 336 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:01:34.976 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:34 smithi089 bash[82202]: cluster 2024-07-15T07:01:34.805437+0000 mgr.y (mgr.25244) 208 : cluster [DBG] pgmap v144: 73 pgs: 10 stale+active+clean, 63 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:01:34.976 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:34 smithi089 bash[79810]: cluster 2024-07-15T07:01:33.881858+0000 mon.a (mon.0) 332 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-07-15T07:01:34.977 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:34 smithi089 bash[79810]: cluster 2024-07-15T07:01:33.881901+0000 mon.a (mon.0) 333 : cluster [ERR] Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-07-15T07:01:34.977 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:34 smithi089 bash[79810]: cluster 2024-07-15T07:01:33.901474+0000 mon.a (mon.0) 334 : cluster [DBG] osdmap e85: 8 total, 7 up, 8 in 2024-07-15T07:01:34.977 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:34 smithi089 bash[79810]: audit 2024-07-15T07:01:34.793657+0000 mon.a (mon.0) 335 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:01:34.977 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:34 smithi089 bash[79810]: audit 2024-07-15T07:01:34.794709+0000 mon.a (mon.0) 336 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:01:34.977 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:34 smithi089 bash[79810]: cluster 2024-07-15T07:01:34.805437+0000 mgr.y (mgr.25244) 208 : cluster [DBG] pgmap v144: 73 pgs: 10 stale+active+clean, 63 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:01:35.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:34 smithi134 bash[67624]: cluster 2024-07-15T07:01:33.881858+0000 mon.a (mon.0) 332 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-07-15T07:01:35.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:34 smithi134 bash[67624]: cluster 2024-07-15T07:01:33.881901+0000 mon.a (mon.0) 333 : cluster [ERR] Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-07-15T07:01:35.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:34 smithi134 bash[67624]: cluster 2024-07-15T07:01:33.901474+0000 mon.a (mon.0) 334 : cluster [DBG] osdmap e85: 8 total, 7 up, 8 in 2024-07-15T07:01:35.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:34 smithi134 bash[67624]: audit 2024-07-15T07:01:34.793657+0000 mon.a (mon.0) 335 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:01:35.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:34 smithi134 bash[67624]: audit 2024-07-15T07:01:34.794709+0000 mon.a (mon.0) 336 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:01:35.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:34 smithi134 bash[67624]: cluster 2024-07-15T07:01:34.805437+0000 mgr.y (mgr.25244) 208 : cluster [DBG] pgmap v144: 73 pgs: 10 stale+active+clean, 63 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:01:35.369 INFO:journalctl@ceph.osd.1.smithi089.stdout:Jul 15 07:01:34 smithi089 bash[92559]: ceph-75ba02bc-4274-11ef-bcad-c7b262605968-osd-1 2024-07-15T07:01:35.869 INFO:journalctl@ceph.osd.1.smithi089.stdout:Jul 15 07:01:35 smithi089 bash[92654]: Error response from daemon: No such container: ceph-75ba02bc-4274-11ef-bcad-c7b262605968-osd-1 2024-07-15T07:01:36.188 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:35 smithi089 bash[82202]: cluster 2024-07-15T07:01:34.906984+0000 mon.a (mon.0) 337 : cluster [DBG] osdmap e86: 8 total, 7 up, 8 in 2024-07-15T07:01:36.189 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:35 smithi089 bash[79810]: cluster 2024-07-15T07:01:34.906984+0000 mon.a (mon.0) 337 : cluster [DBG] osdmap e86: 8 total, 7 up, 8 in 2024-07-15T07:01:36.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:35 smithi134 bash[67624]: cluster 2024-07-15T07:01:34.906984+0000 mon.a (mon.0) 337 : cluster [DBG] osdmap e86: 8 total, 7 up, 8 in 2024-07-15T07:01:37.174 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:36 smithi089 bash[79810]: cluster 2024-07-15T07:01:36.806560+0000 mgr.y (mgr.25244) 209 : cluster [DBG] pgmap v146: 73 pgs: 26 active+undersized, 1 stale+active+clean, 2 active+undersized+degraded, 44 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2024-07-15T07:01:37.175 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:36 smithi089 bash[82202]: cluster 2024-07-15T07:01:36.806560+0000 mgr.y (mgr.25244) 209 : cluster [DBG] pgmap v146: 73 pgs: 26 active+undersized, 1 stale+active+clean, 2 active+undersized+degraded, 44 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2024-07-15T07:01:37.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:36 smithi134 bash[67624]: cluster 2024-07-15T07:01:36.806560+0000 mgr.y (mgr.25244) 209 : cluster [DBG] pgmap v146: 73 pgs: 26 active+undersized, 1 stale+active+clean, 2 active+undersized+degraded, 44 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2024-07-15T07:01:38.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:37 smithi089 bash[82202]: cluster 2024-07-15T07:01:36.900818+0000 mon.a (mon.0) 338 : cluster [WRN] Health check failed: Degraded data redundancy: 2/72 objects degraded (2.778%), 2 pgs degraded (PG_DEGRADED) 2024-07-15T07:01:38.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:37 smithi089 bash[79810]: cluster 2024-07-15T07:01:36.900818+0000 mon.a (mon.0) 338 : cluster [WRN] Health check failed: Degraded data redundancy: 2/72 objects degraded (2.778%), 2 pgs degraded (PG_DEGRADED) 2024-07-15T07:01:38.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:37 smithi134 bash[67624]: cluster 2024-07-15T07:01:36.900818+0000 mon.a (mon.0) 338 : cluster [WRN] Health check failed: Degraded data redundancy: 2/72 objects degraded (2.778%), 2 pgs degraded (PG_DEGRADED) 2024-07-15T07:01:38.677 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:38 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:01:38.677 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:38 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:01:38.677 INFO:journalctl@ceph.osd.1.smithi089.stdout:Jul 15 07:01:38 smithi089 systemd[1]: ceph-75ba02bc-4274-11ef-bcad-c7b262605968@osd.1.service: Deactivated successfully. 2024-07-15T07:01:38.677 INFO:journalctl@ceph.osd.1.smithi089.stdout:Jul 15 07:01:38 smithi089 systemd[1]: Stopped Ceph osd.1 for 75ba02bc-4274-11ef-bcad-c7b262605968. 2024-07-15T07:01:38.677 INFO:journalctl@ceph.osd.1.smithi089.stdout:Jul 15 07:01:38 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:01:38.678 INFO:journalctl@ceph.osd.2.smithi089.stdout:Jul 15 07:01:38 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:01:38.678 INFO:journalctl@ceph.osd.3.smithi089.stdout:Jul 15 07:01:38 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:01:38.678 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 07:01:38 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:01:38.678 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 07:01:38 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:01:38.679 INFO:journalctl@ceph.osd.0.smithi089.stdout:Jul 15 07:01:38 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:01:38.679 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 07:01:38 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:01:39.119 INFO:journalctl@ceph.osd.1.smithi089.stdout:Jul 15 07:01:38 smithi089 systemd[1]: Started Ceph osd.1 for 75ba02bc-4274-11ef-bcad-c7b262605968. 2024-07-15T07:01:39.962 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:39 smithi089 bash[79810]: audit 2024-07-15T07:01:38.716065+0000 mon.a (mon.0) 339 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:01:39.963 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:39 smithi089 bash[79810]: audit 2024-07-15T07:01:38.721508+0000 mon.a (mon.0) 340 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:01:39.963 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:39 smithi089 bash[79810]: audit 2024-07-15T07:01:38.722580+0000 mon.a (mon.0) 341 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T07:01:39.963 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:39 smithi089 bash[79810]: cluster 2024-07-15T07:01:38.807037+0000 mgr.y (mgr.25244) 210 : cluster [DBG] pgmap v147: 73 pgs: 29 active+undersized, 3 active+undersized+degraded, 41 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2024-07-15T07:01:39.963 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:39 smithi089 bash[82202]: audit 2024-07-15T07:01:38.716065+0000 mon.a (mon.0) 339 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:01:39.963 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:39 smithi089 bash[82202]: audit 2024-07-15T07:01:38.721508+0000 mon.a (mon.0) 340 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:01:39.963 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:39 smithi089 bash[82202]: audit 2024-07-15T07:01:38.722580+0000 mon.a (mon.0) 341 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T07:01:39.963 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:39 smithi089 bash[82202]: cluster 2024-07-15T07:01:38.807037+0000 mgr.y (mgr.25244) 210 : cluster [DBG] pgmap v147: 73 pgs: 29 active+undersized, 3 active+undersized+degraded, 41 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2024-07-15T07:01:40.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:39 smithi134 bash[67624]: audit 2024-07-15T07:01:38.716065+0000 mon.a (mon.0) 339 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:01:40.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:39 smithi134 bash[67624]: audit 2024-07-15T07:01:38.721508+0000 mon.a (mon.0) 340 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:01:40.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:39 smithi134 bash[67624]: audit 2024-07-15T07:01:38.722580+0000 mon.a (mon.0) 341 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T07:01:40.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:39 smithi134 bash[67624]: cluster 2024-07-15T07:01:38.807037+0000 mgr.y (mgr.25244) 210 : cluster [DBG] pgmap v147: 73 pgs: 29 active+undersized, 3 active+undersized+degraded, 41 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2024-07-15T07:01:40.663 INFO:journalctl@ceph.osd.1.smithi089.stdout:Jul 15 07:01:40 smithi089 bash[92845]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-07-15T07:01:40.663 INFO:journalctl@ceph.osd.1.smithi089.stdout:Jul 15 07:01:40 smithi089 bash[92845]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-07-15T07:01:40.663 INFO:journalctl@ceph.osd.1.smithi089.stdout:Jul 15 07:01:40 smithi089 bash[92845]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-07-15T07:01:40.663 INFO:journalctl@ceph.osd.1.smithi089.stdout:Jul 15 07:01:40 smithi089 bash[92845]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-07-15T07:01:40.663 INFO:journalctl@ceph.osd.1.smithi089.stdout:Jul 15 07:01:40 smithi089 bash[92845]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-07-15T07:01:40.663 INFO:journalctl@ceph.osd.1.smithi089.stdout:Jul 15 07:01:40 smithi089 bash[92845]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-07-15T07:01:40.664 INFO:journalctl@ceph.osd.1.smithi089.stdout:Jul 15 07:01:40 smithi089 bash[92845]: --> ceph-volume raw activate successful for osd ID: 1 2024-07-15T07:01:41.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:40 smithi089 bash[79810]: cluster 2024-07-15T07:01:40.808029+0000 mgr.y (mgr.25244) 211 : cluster [DBG] pgmap v148: 73 pgs: 29 active+undersized, 3 active+undersized+degraded, 41 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2024-07-15T07:01:41.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:40 smithi089 bash[82202]: cluster 2024-07-15T07:01:40.808029+0000 mgr.y (mgr.25244) 211 : cluster [DBG] pgmap v148: 73 pgs: 29 active+undersized, 3 active+undersized+degraded, 41 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2024-07-15T07:01:41.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:40 smithi134 bash[67624]: cluster 2024-07-15T07:01:40.808029+0000 mgr.y (mgr.25244) 211 : cluster [DBG] pgmap v148: 73 pgs: 29 active+undersized, 3 active+undersized+degraded, 41 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2024-07-15T07:01:42.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:42 smithi089 bash[79810]: cluster 2024-07-15T07:01:42.279275+0000 mon.a (mon.0) 342 : cluster [WRN] Health check update: Degraded data redundancy: 3/72 objects degraded (4.167%), 3 pgs degraded (PG_DEGRADED) 2024-07-15T07:01:42.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:42 smithi089 bash[82202]: cluster 2024-07-15T07:01:42.279275+0000 mon.a (mon.0) 342 : cluster [WRN] Health check update: Degraded data redundancy: 3/72 objects degraded (4.167%), 3 pgs degraded (PG_DEGRADED) 2024-07-15T07:01:42.620 INFO:journalctl@ceph.osd.1.smithi089.stdout:Jul 15 07:01:42 smithi089 bash[93078]: debug 2024-07-15T07:01:42.385+0000 7f7bf6655640 1 -- 172.21.15.89:0/400520747 <== mon.2 v2:172.21.15.89:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x561b2a3491e0 con 0x561b2a309c00 2024-07-15T07:01:42.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:42 smithi134 bash[67624]: cluster 2024-07-15T07:01:42.279275+0000 mon.a (mon.0) 342 : cluster [WRN] Health check update: Degraded data redundancy: 3/72 objects degraded (4.167%), 3 pgs degraded (PG_DEGRADED) 2024-07-15T07:01:42.805 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:01:42 smithi134 bash[60729]: ts=2024-07-15T07:01:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T07:01:43.337 INFO:journalctl@ceph.osd.1.smithi089.stdout:Jul 15 07:01:42 smithi089 bash[93078]: debug 2024-07-15T07:01:42.921+0000 7f7bf8ebb640 -1 Falling back to public interface 2024-07-15T07:01:43.619 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:43 smithi089 bash[82202]: cluster 2024-07-15T07:01:42.808737+0000 mgr.y (mgr.25244) 212 : cluster [DBG] pgmap v149: 73 pgs: 29 active+undersized, 3 active+undersized+degraded, 41 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2024-07-15T07:01:43.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:43 smithi089 bash[79810]: cluster 2024-07-15T07:01:42.808737+0000 mgr.y (mgr.25244) 212 : cluster [DBG] pgmap v149: 73 pgs: 29 active+undersized, 3 active+undersized+degraded, 41 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2024-07-15T07:01:43.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:43 smithi134 bash[67624]: cluster 2024-07-15T07:01:42.808737+0000 mgr.y (mgr.25244) 212 : cluster [DBG] pgmap v149: 73 pgs: 29 active+undersized, 3 active+undersized+degraded, 41 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2024-07-15T07:01:44.619 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 07:01:44 smithi089 bash[72894]: ::ffff:172.21.15.134 - - [15/Jul/2024:07:01:44] "GET /metrics HTTP/1.1" 200 33989 "" "Prometheus/2.43.0" 2024-07-15T07:01:45.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:44 smithi134 bash[67624]: cluster 2024-07-15T07:01:44.809268+0000 mgr.y (mgr.25244) 213 : cluster [DBG] pgmap v150: 73 pgs: 29 active+undersized, 3 active+undersized+degraded, 41 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2024-07-15T07:01:45.369 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:44 smithi089 bash[79810]: cluster 2024-07-15T07:01:44.809268+0000 mgr.y (mgr.25244) 213 : cluster [DBG] pgmap v150: 73 pgs: 29 active+undersized, 3 active+undersized+degraded, 41 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2024-07-15T07:01:45.369 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:44 smithi089 bash[82202]: cluster 2024-07-15T07:01:44.809268+0000 mgr.y (mgr.25244) 213 : cluster [DBG] pgmap v150: 73 pgs: 29 active+undersized, 3 active+undersized+degraded, 41 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2024-07-15T07:01:46.339 INFO:journalctl@ceph.osd.1.smithi089.stdout:Jul 15 07:01:45 smithi089 bash[93078]: debug 2024-07-15T07:01:45.952+0000 7f7bf8ebb640 -1 osd.1 84 log_to_monitors true 2024-07-15T07:01:46.619 INFO:journalctl@ceph.osd.1.smithi089.stdout:Jul 15 07:01:46 smithi089 bash[93078]: debug 2024-07-15T07:01:46.396+0000 7f7bf0c65640 -1 osd.1 84 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-07-15T07:01:46.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:46 smithi089 bash[79810]: audit 2024-07-15T07:01:45.342252+0000 mon.a (mon.0) 343 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:01:46.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:46 smithi089 bash[79810]: audit 2024-07-15T07:01:45.349270+0000 mon.a (mon.0) 344 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:01:46.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:46 smithi089 bash[79810]: audit 2024-07-15T07:01:45.961936+0000 mon.c (mon.2) 4 : audit [INF] from='osd.1 [v2:172.21.15.89:6810/539615257,v1:172.21.15.89:6811/539615257]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-07-15T07:01:46.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:46 smithi089 bash[79810]: audit 2024-07-15T07:01:45.962219+0000 mon.a (mon.0) 345 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-07-15T07:01:46.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:46 smithi089 bash[79810]: audit 2024-07-15T07:01:45.998139+0000 mon.a (mon.0) 346 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:01:46.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:46 smithi089 bash[79810]: audit 2024-07-15T07:01:46.003715+0000 mon.a (mon.0) 347 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:01:46.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:46 smithi089 bash[82202]: audit 2024-07-15T07:01:45.342252+0000 mon.a (mon.0) 343 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:01:46.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:46 smithi089 bash[82202]: audit 2024-07-15T07:01:45.349270+0000 mon.a (mon.0) 344 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:01:46.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:46 smithi089 bash[82202]: audit 2024-07-15T07:01:45.961936+0000 mon.c (mon.2) 4 : audit [INF] from='osd.1 [v2:172.21.15.89:6810/539615257,v1:172.21.15.89:6811/539615257]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-07-15T07:01:46.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:46 smithi089 bash[82202]: audit 2024-07-15T07:01:45.962219+0000 mon.a (mon.0) 345 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-07-15T07:01:46.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:46 smithi089 bash[82202]: audit 2024-07-15T07:01:45.998139+0000 mon.a (mon.0) 346 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:01:46.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:46 smithi089 bash[82202]: audit 2024-07-15T07:01:46.003715+0000 mon.a (mon.0) 347 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:01:46.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:46 smithi134 bash[67624]: audit 2024-07-15T07:01:45.342252+0000 mon.a (mon.0) 343 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:01:46.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:46 smithi134 bash[67624]: audit 2024-07-15T07:01:45.349270+0000 mon.a (mon.0) 344 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:01:46.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:46 smithi134 bash[67624]: audit 2024-07-15T07:01:45.961936+0000 mon.c (mon.2) 4 : audit [INF] from='osd.1 [v2:172.21.15.89:6810/539615257,v1:172.21.15.89:6811/539615257]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-07-15T07:01:46.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:46 smithi134 bash[67624]: audit 2024-07-15T07:01:45.962219+0000 mon.a (mon.0) 345 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-07-15T07:01:46.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:46 smithi134 bash[67624]: audit 2024-07-15T07:01:45.998139+0000 mon.a (mon.0) 346 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:01:46.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:46 smithi134 bash[67624]: audit 2024-07-15T07:01:46.003715+0000 mon.a (mon.0) 347 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:01:47.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:47 smithi089 bash[82202]: audit 2024-07-15T07:01:46.362186+0000 mon.a (mon.0) 348 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-07-15T07:01:47.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:47 smithi089 bash[82202]: cluster 2024-07-15T07:01:46.362302+0000 mon.a (mon.0) 349 : cluster [DBG] osdmap e87: 8 total, 7 up, 8 in 2024-07-15T07:01:47.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:47 smithi089 bash[82202]: audit 2024-07-15T07:01:46.363833+0000 mon.c (mon.2) 5 : audit [INF] from='osd.1 [v2:172.21.15.89:6810/539615257,v1:172.21.15.89:6811/539615257]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2024-07-15T07:01:47.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:47 smithi089 bash[82202]: audit 2024-07-15T07:01:46.364572+0000 mon.a (mon.0) 350 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2024-07-15T07:01:47.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:47 smithi089 bash[82202]: cluster 2024-07-15T07:01:46.810075+0000 mgr.y (mgr.25244) 214 : cluster [DBG] pgmap v152: 73 pgs: 29 active+undersized, 3 active+undersized+degraded, 41 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2024-07-15T07:01:47.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:47 smithi089 bash[79810]: debug 2024-07-15T07:01:47.352+0000 7fb563a0a640 -1 log_channel(cluster) log [ERR] : Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-07-15T07:01:47.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:47 smithi089 bash[79810]: audit 2024-07-15T07:01:46.362186+0000 mon.a (mon.0) 348 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-07-15T07:01:47.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:47 smithi089 bash[79810]: cluster 2024-07-15T07:01:46.362302+0000 mon.a (mon.0) 349 : cluster [DBG] osdmap e87: 8 total, 7 up, 8 in 2024-07-15T07:01:47.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:47 smithi089 bash[79810]: audit 2024-07-15T07:01:46.363833+0000 mon.c (mon.2) 5 : audit [INF] from='osd.1 [v2:172.21.15.89:6810/539615257,v1:172.21.15.89:6811/539615257]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2024-07-15T07:01:47.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:47 smithi089 bash[79810]: audit 2024-07-15T07:01:46.364572+0000 mon.a (mon.0) 350 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2024-07-15T07:01:47.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:47 smithi089 bash[79810]: cluster 2024-07-15T07:01:46.810075+0000 mgr.y (mgr.25244) 214 : cluster [DBG] pgmap v152: 73 pgs: 29 active+undersized, 3 active+undersized+degraded, 41 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2024-07-15T07:01:47.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:47 smithi134 bash[67624]: audit 2024-07-15T07:01:46.362186+0000 mon.a (mon.0) 348 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-07-15T07:01:47.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:47 smithi134 bash[67624]: cluster 2024-07-15T07:01:46.362302+0000 mon.a (mon.0) 349 : cluster [DBG] osdmap e87: 8 total, 7 up, 8 in 2024-07-15T07:01:47.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:47 smithi134 bash[67624]: audit 2024-07-15T07:01:46.363833+0000 mon.c (mon.2) 5 : audit [INF] from='osd.1 [v2:172.21.15.89:6810/539615257,v1:172.21.15.89:6811/539615257]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2024-07-15T07:01:47.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:47 smithi134 bash[67624]: audit 2024-07-15T07:01:46.364572+0000 mon.a (mon.0) 350 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2024-07-15T07:01:47.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:47 smithi134 bash[67624]: cluster 2024-07-15T07:01:46.810075+0000 mgr.y (mgr.25244) 214 : cluster [DBG] pgmap v152: 73 pgs: 29 active+undersized, 3 active+undersized+degraded, 41 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2024-07-15T07:01:48.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:48 smithi089 bash[79810]: cluster 2024-07-15T07:01:47.355554+0000 mon.a (mon.0) 351 : cluster [ERR] Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-07-15T07:01:48.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:48 smithi089 bash[79810]: cluster 2024-07-15T07:01:47.355618+0000 mon.a (mon.0) 352 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-07-15T07:01:48.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:48 smithi089 bash[79810]: cluster 2024-07-15T07:01:47.364445+0000 mon.a (mon.0) 353 : cluster [INF] osd.1 [v2:172.21.15.89:6810/539615257,v1:172.21.15.89:6811/539615257] boot 2024-07-15T07:01:48.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:48 smithi089 bash[79810]: cluster 2024-07-15T07:01:47.364606+0000 mon.a (mon.0) 354 : cluster [DBG] osdmap e88: 8 total, 8 up, 8 in 2024-07-15T07:01:48.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:48 smithi089 bash[79810]: audit 2024-07-15T07:01:47.365535+0000 mon.a (mon.0) 355 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-07-15T07:01:48.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:48 smithi089 bash[82202]: cluster 2024-07-15T07:01:47.355554+0000 mon.a (mon.0) 351 : cluster [ERR] Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-07-15T07:01:48.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:48 smithi089 bash[82202]: cluster 2024-07-15T07:01:47.355618+0000 mon.a (mon.0) 352 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-07-15T07:01:48.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:48 smithi089 bash[82202]: cluster 2024-07-15T07:01:47.364445+0000 mon.a (mon.0) 353 : cluster [INF] osd.1 [v2:172.21.15.89:6810/539615257,v1:172.21.15.89:6811/539615257] boot 2024-07-15T07:01:48.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:48 smithi089 bash[82202]: cluster 2024-07-15T07:01:47.364606+0000 mon.a (mon.0) 354 : cluster [DBG] osdmap e88: 8 total, 8 up, 8 in 2024-07-15T07:01:48.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:48 smithi089 bash[82202]: audit 2024-07-15T07:01:47.365535+0000 mon.a (mon.0) 355 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-07-15T07:01:48.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:48 smithi134 bash[67624]: cluster 2024-07-15T07:01:47.355554+0000 mon.a (mon.0) 351 : cluster [ERR] Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-07-15T07:01:48.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:48 smithi134 bash[67624]: cluster 2024-07-15T07:01:47.355618+0000 mon.a (mon.0) 352 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-07-15T07:01:48.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:48 smithi134 bash[67624]: cluster 2024-07-15T07:01:47.364445+0000 mon.a (mon.0) 353 : cluster [INF] osd.1 [v2:172.21.15.89:6810/539615257,v1:172.21.15.89:6811/539615257] boot 2024-07-15T07:01:48.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:48 smithi134 bash[67624]: cluster 2024-07-15T07:01:47.364606+0000 mon.a (mon.0) 354 : cluster [DBG] osdmap e88: 8 total, 8 up, 8 in 2024-07-15T07:01:48.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:48 smithi134 bash[67624]: audit 2024-07-15T07:01:47.365535+0000 mon.a (mon.0) 355 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-07-15T07:01:49.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:49 smithi134 bash[67624]: cluster 2024-07-15T07:01:48.389067+0000 mon.a (mon.0) 356 : cluster [DBG] osdmap e89: 8 total, 8 up, 8 in 2024-07-15T07:01:49.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:49 smithi134 bash[67624]: cluster 2024-07-15T07:01:48.810733+0000 mgr.y (mgr.25244) 215 : cluster [DBG] pgmap v155: 73 pgs: 3 peering, 27 active+undersized, 2 active+undersized+degraded, 41 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2024-07-15T07:01:49.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:49 smithi089 bash[79810]: cluster 2024-07-15T07:01:48.389067+0000 mon.a (mon.0) 356 : cluster [DBG] osdmap e89: 8 total, 8 up, 8 in 2024-07-15T07:01:49.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:49 smithi089 bash[79810]: cluster 2024-07-15T07:01:48.810733+0000 mgr.y (mgr.25244) 215 : cluster [DBG] pgmap v155: 73 pgs: 3 peering, 27 active+undersized, 2 active+undersized+degraded, 41 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2024-07-15T07:01:49.870 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:49 smithi089 bash[82202]: cluster 2024-07-15T07:01:48.389067+0000 mon.a (mon.0) 356 : cluster [DBG] osdmap e89: 8 total, 8 up, 8 in 2024-07-15T07:01:49.870 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:49 smithi089 bash[82202]: cluster 2024-07-15T07:01:48.810733+0000 mgr.y (mgr.25244) 215 : cluster [DBG] pgmap v155: 73 pgs: 3 peering, 27 active+undersized, 2 active+undersized+degraded, 41 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2024-07-15T07:01:50.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:50 smithi134 bash[67624]: cluster 2024-07-15T07:01:49.382480+0000 mon.a (mon.0) 357 : cluster [WRN] Health check update: Degraded data redundancy: 2/72 objects degraded (2.778%), 2 pgs degraded (PG_DEGRADED) 2024-07-15T07:01:50.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:50 smithi134 bash[67624]: audit 2024-07-15T07:01:49.796796+0000 mon.a (mon.0) 358 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:01:50.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:50 smithi134 bash[67624]: audit 2024-07-15T07:01:49.798735+0000 mon.a (mon.0) 359 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:01:50.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:50 smithi089 bash[79810]: cluster 2024-07-15T07:01:49.382480+0000 mon.a (mon.0) 357 : cluster [WRN] Health check update: Degraded data redundancy: 2/72 objects degraded (2.778%), 2 pgs degraded (PG_DEGRADED) 2024-07-15T07:01:50.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:50 smithi089 bash[79810]: audit 2024-07-15T07:01:49.796796+0000 mon.a (mon.0) 358 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:01:50.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:50 smithi089 bash[79810]: audit 2024-07-15T07:01:49.798735+0000 mon.a (mon.0) 359 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:01:50.870 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:50 smithi089 bash[82202]: cluster 2024-07-15T07:01:49.382480+0000 mon.a (mon.0) 357 : cluster [WRN] Health check update: Degraded data redundancy: 2/72 objects degraded (2.778%), 2 pgs degraded (PG_DEGRADED) 2024-07-15T07:01:50.870 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:50 smithi089 bash[82202]: audit 2024-07-15T07:01:49.796796+0000 mon.a (mon.0) 358 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:01:50.870 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:50 smithi089 bash[82202]: audit 2024-07-15T07:01:49.798735+0000 mon.a (mon.0) 359 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:01:51.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:51 smithi134 bash[67624]: cluster 2024-07-15T07:01:50.811799+0000 mgr.y (mgr.25244) 216 : cluster [DBG] pgmap v156: 73 pgs: 3 peering, 10 active+undersized, 60 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:01:51.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:51 smithi089 bash[79810]: cluster 2024-07-15T07:01:50.811799+0000 mgr.y (mgr.25244) 216 : cluster [DBG] pgmap v156: 73 pgs: 3 peering, 10 active+undersized, 60 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:01:51.870 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:51 smithi089 bash[82202]: cluster 2024-07-15T07:01:50.811799+0000 mgr.y (mgr.25244) 216 : cluster [DBG] pgmap v156: 73 pgs: 3 peering, 10 active+undersized, 60 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:01:52.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:52 smithi134 bash[67624]: cluster 2024-07-15T07:01:51.395498+0000 mon.a (mon.0) 360 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 2 pgs degraded) 2024-07-15T07:01:52.805 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:01:52 smithi134 bash[60729]: ts=2024-07-15T07:01:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T07:01:52.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:52 smithi089 bash[79810]: cluster 2024-07-15T07:01:51.395498+0000 mon.a (mon.0) 360 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 2 pgs degraded) 2024-07-15T07:01:52.870 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:52 smithi089 bash[82202]: cluster 2024-07-15T07:01:51.395498+0000 mon.a (mon.0) 360 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 2 pgs degraded) 2024-07-15T07:01:53.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:53 smithi134 bash[67624]: cluster 2024-07-15T07:01:52.812662+0000 mgr.y (mgr.25244) 217 : cluster [DBG] pgmap v157: 73 pgs: 3 peering, 70 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:01:53.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:53 smithi089 bash[79810]: cluster 2024-07-15T07:01:52.812662+0000 mgr.y (mgr.25244) 217 : cluster [DBG] pgmap v157: 73 pgs: 3 peering, 70 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:01:53.870 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:53 smithi089 bash[82202]: cluster 2024-07-15T07:01:52.812662+0000 mgr.y (mgr.25244) 217 : cluster [DBG] pgmap v157: 73 pgs: 3 peering, 70 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:01:54.620 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 07:01:54 smithi089 bash[72894]: ::ffff:172.21.15.134 - - [15/Jul/2024:07:01:54] "GET /metrics HTTP/1.1" 200 33843 "" "Prometheus/2.43.0" 2024-07-15T07:01:55.169 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:54 smithi089 bash[79810]: cluster 2024-07-15T07:01:54.813241+0000 mgr.y (mgr.25244) 218 : cluster [DBG] pgmap v158: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:01:55.170 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:54 smithi089 bash[82202]: cluster 2024-07-15T07:01:54.813241+0000 mgr.y (mgr.25244) 218 : cluster [DBG] pgmap v158: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:01:55.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:54 smithi134 bash[67624]: cluster 2024-07-15T07:01:54.813241+0000 mgr.y (mgr.25244) 218 : cluster [DBG] pgmap v158: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:01:57.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:56 smithi089 bash[79810]: audit 2024-07-15T07:01:55.867565+0000 mon.a (mon.0) 361 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:01:57.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:56 smithi089 bash[79810]: audit 2024-07-15T07:01:55.879257+0000 mon.a (mon.0) 362 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:01:57.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:56 smithi089 bash[79810]: audit 2024-07-15T07:01:55.881422+0000 mon.a (mon.0) 363 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T07:01:57.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:56 smithi089 bash[79810]: audit 2024-07-15T07:01:55.882789+0000 mon.a (mon.0) 364 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T07:01:57.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:56 smithi089 bash[79810]: audit 2024-07-15T07:01:55.893782+0000 mon.a (mon.0) 365 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:01:57.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:56 smithi089 bash[79810]: audit 2024-07-15T07:01:56.047334+0000 mon.a (mon.0) 366 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T07:01:57.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:56 smithi089 bash[79810]: cluster 2024-07-15T07:01:56.814313+0000 mgr.y (mgr.25244) 219 : cluster [DBG] pgmap v159: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:01:57.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:56 smithi089 bash[82202]: audit 2024-07-15T07:01:55.867565+0000 mon.a (mon.0) 361 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:01:57.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:56 smithi089 bash[82202]: audit 2024-07-15T07:01:55.879257+0000 mon.a (mon.0) 362 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:01:57.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:56 smithi089 bash[82202]: audit 2024-07-15T07:01:55.881422+0000 mon.a (mon.0) 363 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T07:01:57.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:56 smithi089 bash[82202]: audit 2024-07-15T07:01:55.882789+0000 mon.a (mon.0) 364 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T07:01:57.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:56 smithi089 bash[82202]: audit 2024-07-15T07:01:55.893782+0000 mon.a (mon.0) 365 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:01:57.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:56 smithi089 bash[82202]: audit 2024-07-15T07:01:56.047334+0000 mon.a (mon.0) 366 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T07:01:57.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:56 smithi089 bash[82202]: cluster 2024-07-15T07:01:56.814313+0000 mgr.y (mgr.25244) 219 : cluster [DBG] pgmap v159: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:01:57.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:56 smithi134 bash[67624]: audit 2024-07-15T07:01:55.867565+0000 mon.a (mon.0) 361 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:01:57.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:56 smithi134 bash[67624]: audit 2024-07-15T07:01:55.879257+0000 mon.a (mon.0) 362 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:01:57.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:56 smithi134 bash[67624]: audit 2024-07-15T07:01:55.881422+0000 mon.a (mon.0) 363 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T07:01:57.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:56 smithi134 bash[67624]: audit 2024-07-15T07:01:55.882789+0000 mon.a (mon.0) 364 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T07:01:57.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:56 smithi134 bash[67624]: audit 2024-07-15T07:01:55.893782+0000 mon.a (mon.0) 365 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:01:57.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:56 smithi134 bash[67624]: audit 2024-07-15T07:01:56.047334+0000 mon.a (mon.0) 366 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T07:01:57.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:56 smithi134 bash[67624]: cluster 2024-07-15T07:01:56.814313+0000 mgr.y (mgr.25244) 219 : cluster [DBG] pgmap v159: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:01:59.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:01:58 smithi134 bash[67624]: cluster 2024-07-15T07:01:58.815174+0000 mgr.y (mgr.25244) 220 : cluster [DBG] pgmap v160: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:01:59.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:01:58 smithi089 bash[79810]: cluster 2024-07-15T07:01:58.815174+0000 mgr.y (mgr.25244) 220 : cluster [DBG] pgmap v160: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:01:59.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:01:58 smithi089 bash[82202]: cluster 2024-07-15T07:01:58.815174+0000 mgr.y (mgr.25244) 220 : cluster [DBG] pgmap v160: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:02:01.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:02:00 smithi134 bash[67624]: audit 2024-07-15T07:01:59.891423+0000 mon.a (mon.0) 367 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:02:01.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:02:00 smithi134 bash[67624]: cluster 2024-07-15T07:02:00.816258+0000 mgr.y (mgr.25244) 221 : cluster [DBG] pgmap v161: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:02:01.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:02:00 smithi089 bash[79810]: audit 2024-07-15T07:01:59.891423+0000 mon.a (mon.0) 367 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:02:01.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:02:00 smithi089 bash[79810]: cluster 2024-07-15T07:02:00.816258+0000 mgr.y (mgr.25244) 221 : cluster [DBG] pgmap v161: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:02:01.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:02:00 smithi089 bash[82202]: audit 2024-07-15T07:01:59.891423+0000 mon.a (mon.0) 367 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:02:01.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:02:00 smithi089 bash[82202]: cluster 2024-07-15T07:02:00.816258+0000 mgr.y (mgr.25244) 221 : cluster [DBG] pgmap v161: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:02:02.804 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:02:02 smithi134 bash[60729]: ts=2024-07-15T07:02:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T07:02:03.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:02:02 smithi089 bash[79810]: audit 2024-07-15T07:02:01.859604+0000 mon.a (mon.0) 368 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:02:03.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:02:02 smithi089 bash[79810]: audit 2024-07-15T07:02:01.869569+0000 mon.a (mon.0) 369 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:02:03.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:02:02 smithi089 bash[79810]: audit 2024-07-15T07:02:02.351460+0000 mon.a (mon.0) 370 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T07:02:03.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:02:02 smithi089 bash[79810]: audit 2024-07-15T07:02:02.353133+0000 mon.a (mon.0) 371 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T07:02:03.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:02:02 smithi089 bash[79810]: audit 2024-07-15T07:02:02.363495+0000 mon.a (mon.0) 372 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:02:03.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:02:02 smithi089 bash[79810]: cluster 2024-07-15T07:02:02.816987+0000 mgr.y (mgr.25244) 222 : cluster [DBG] pgmap v162: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:02:03.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:02:02 smithi089 bash[82202]: audit 2024-07-15T07:02:01.859604+0000 mon.a (mon.0) 368 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:02:03.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:02:02 smithi089 bash[82202]: audit 2024-07-15T07:02:01.869569+0000 mon.a (mon.0) 369 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:02:03.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:02:02 smithi089 bash[82202]: audit 2024-07-15T07:02:02.351460+0000 mon.a (mon.0) 370 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T07:02:03.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:02:02 smithi089 bash[82202]: audit 2024-07-15T07:02:02.353133+0000 mon.a (mon.0) 371 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T07:02:03.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:02:02 smithi089 bash[82202]: audit 2024-07-15T07:02:02.363495+0000 mon.a (mon.0) 372 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:02:03.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:02:02 smithi089 bash[82202]: cluster 2024-07-15T07:02:02.816987+0000 mgr.y (mgr.25244) 222 : cluster [DBG] pgmap v162: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:02:03.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:02:02 smithi134 bash[67624]: audit 2024-07-15T07:02:01.859604+0000 mon.a (mon.0) 368 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:02:03.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:02:02 smithi134 bash[67624]: audit 2024-07-15T07:02:01.869569+0000 mon.a (mon.0) 369 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:02:03.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:02:02 smithi134 bash[67624]: audit 2024-07-15T07:02:02.351460+0000 mon.a (mon.0) 370 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T07:02:03.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:02:02 smithi134 bash[67624]: audit 2024-07-15T07:02:02.353133+0000 mon.a (mon.0) 371 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T07:02:03.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:02:02 smithi134 bash[67624]: audit 2024-07-15T07:02:02.363495+0000 mon.a (mon.0) 372 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:02:03.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:02:02 smithi134 bash[67624]: cluster 2024-07-15T07:02:02.816987+0000 mgr.y (mgr.25244) 222 : cluster [DBG] pgmap v162: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:02:04.620 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 07:02:04 smithi089 bash[72894]: ::ffff:172.21.15.134 - - [15/Jul/2024:07:02:04] "GET /metrics HTTP/1.1" 200 33843 "" "Prometheus/2.43.0" 2024-07-15T07:02:06.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:02:05 smithi134 bash[67624]: audit 2024-07-15T07:02:04.797099+0000 mon.a (mon.0) 373 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:02:06.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:02:05 smithi134 bash[67624]: audit 2024-07-15T07:02:04.798941+0000 mon.a (mon.0) 374 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:02:06.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:02:05 smithi134 bash[67624]: cluster 2024-07-15T07:02:04.817720+0000 mgr.y (mgr.25244) 223 : cluster [DBG] pgmap v163: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:02:06.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:02:05 smithi089 bash[79810]: audit 2024-07-15T07:02:04.797099+0000 mon.a (mon.0) 373 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:02:06.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:02:05 smithi089 bash[79810]: audit 2024-07-15T07:02:04.798941+0000 mon.a (mon.0) 374 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:02:06.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:02:05 smithi089 bash[79810]: cluster 2024-07-15T07:02:04.817720+0000 mgr.y (mgr.25244) 223 : cluster [DBG] pgmap v163: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:02:06.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:02:05 smithi089 bash[82202]: audit 2024-07-15T07:02:04.797099+0000 mon.a (mon.0) 373 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:02:06.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:02:05 smithi089 bash[82202]: audit 2024-07-15T07:02:04.798941+0000 mon.a (mon.0) 374 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:02:06.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:02:05 smithi089 bash[82202]: cluster 2024-07-15T07:02:04.817720+0000 mgr.y (mgr.25244) 223 : cluster [DBG] pgmap v163: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:02:07.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:02:06 smithi134 bash[67624]: cluster 2024-07-15T07:02:06.818856+0000 mgr.y (mgr.25244) 224 : cluster [DBG] pgmap v164: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:02:07.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:02:06 smithi089 bash[79810]: cluster 2024-07-15T07:02:06.818856+0000 mgr.y (mgr.25244) 224 : cluster [DBG] pgmap v164: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:02:07.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:02:06 smithi089 bash[82202]: cluster 2024-07-15T07:02:06.818856+0000 mgr.y (mgr.25244) 224 : cluster [DBG] pgmap v164: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:02:09.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:02:08 smithi134 bash[67624]: cluster 2024-07-15T07:02:08.819720+0000 mgr.y (mgr.25244) 225 : cluster [DBG] pgmap v165: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:02:09.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:02:08 smithi089 bash[79810]: cluster 2024-07-15T07:02:08.819720+0000 mgr.y (mgr.25244) 225 : cluster [DBG] pgmap v165: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:02:09.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:02:08 smithi089 bash[82202]: cluster 2024-07-15T07:02:08.819720+0000 mgr.y (mgr.25244) 225 : cluster [DBG] pgmap v165: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:02:11.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:02:10 smithi134 bash[67624]: cluster 2024-07-15T07:02:10.820791+0000 mgr.y (mgr.25244) 226 : cluster [DBG] pgmap v166: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:02:11.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:02:10 smithi089 bash[79810]: cluster 2024-07-15T07:02:10.820791+0000 mgr.y (mgr.25244) 226 : cluster [DBG] pgmap v166: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:02:11.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:02:10 smithi089 bash[82202]: cluster 2024-07-15T07:02:10.820791+0000 mgr.y (mgr.25244) 226 : cluster [DBG] pgmap v166: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:02:12.804 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:02:12 smithi134 bash[60729]: ts=2024-07-15T07:02:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T07:02:13.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:02:12 smithi134 bash[67624]: cluster 2024-07-15T07:02:12.821635+0000 mgr.y (mgr.25244) 227 : cluster [DBG] pgmap v167: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:02:13.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:02:12 smithi089 bash[79810]: cluster 2024-07-15T07:02:12.821635+0000 mgr.y (mgr.25244) 227 : cluster [DBG] pgmap v167: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:02:13.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:02:12 smithi089 bash[82202]: cluster 2024-07-15T07:02:12.821635+0000 mgr.y (mgr.25244) 227 : cluster [DBG] pgmap v167: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:02:14.620 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 07:02:14 smithi089 bash[72894]: ::ffff:172.21.15.134 - - [15/Jul/2024:07:02:14] "GET /metrics HTTP/1.1" 200 33986 "" "Prometheus/2.43.0" 2024-07-15T07:02:15.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:02:14 smithi134 bash[67624]: cluster 2024-07-15T07:02:14.822487+0000 mgr.y (mgr.25244) 228 : cluster [DBG] pgmap v168: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:02:15.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:02:14 smithi089 bash[79810]: cluster 2024-07-15T07:02:14.822487+0000 mgr.y (mgr.25244) 228 : cluster [DBG] pgmap v168: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:02:15.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:02:14 smithi089 bash[82202]: cluster 2024-07-15T07:02:14.822487+0000 mgr.y (mgr.25244) 228 : cluster [DBG] pgmap v168: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:02:17.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:02:16 smithi134 bash[67624]: cluster 2024-07-15T07:02:16.823573+0000 mgr.y (mgr.25244) 229 : cluster [DBG] pgmap v169: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:02:17.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:02:16 smithi089 bash[79810]: cluster 2024-07-15T07:02:16.823573+0000 mgr.y (mgr.25244) 229 : cluster [DBG] pgmap v169: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:02:17.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:02:16 smithi089 bash[82202]: cluster 2024-07-15T07:02:16.823573+0000 mgr.y (mgr.25244) 229 : cluster [DBG] pgmap v169: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:02:19.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:02:18 smithi134 bash[67624]: cluster 2024-07-15T07:02:18.824242+0000 mgr.y (mgr.25244) 230 : cluster [DBG] pgmap v170: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:02:19.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:02:18 smithi089 bash[79810]: cluster 2024-07-15T07:02:18.824242+0000 mgr.y (mgr.25244) 230 : cluster [DBG] pgmap v170: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:02:19.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:02:18 smithi089 bash[82202]: cluster 2024-07-15T07:02:18.824242+0000 mgr.y (mgr.25244) 230 : cluster [DBG] pgmap v170: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:02:20.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:02:19 smithi134 bash[67624]: audit 2024-07-15T07:02:19.791732+0000 mon.a (mon.0) 375 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:02:20.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:02:19 smithi089 bash[79810]: audit 2024-07-15T07:02:19.791732+0000 mon.a (mon.0) 375 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:02:20.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:02:19 smithi089 bash[82202]: audit 2024-07-15T07:02:19.791732+0000 mon.a (mon.0) 375 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:02:21.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:02:20 smithi134 bash[67624]: cluster 2024-07-15T07:02:20.825224+0000 mgr.y (mgr.25244) 231 : cluster [DBG] pgmap v171: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:02:21.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:02:20 smithi089 bash[79810]: cluster 2024-07-15T07:02:20.825224+0000 mgr.y (mgr.25244) 231 : cluster [DBG] pgmap v171: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:02:21.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:02:20 smithi089 bash[82202]: cluster 2024-07-15T07:02:20.825224+0000 mgr.y (mgr.25244) 231 : cluster [DBG] pgmap v171: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:02:22.804 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:02:22 smithi134 bash[60729]: ts=2024-07-15T07:02:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T07:02:23.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:02:22 smithi134 bash[67624]: cluster 2024-07-15T07:02:22.825916+0000 mgr.y (mgr.25244) 232 : cluster [DBG] pgmap v172: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:02:23.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:02:22 smithi089 bash[79810]: cluster 2024-07-15T07:02:22.825916+0000 mgr.y (mgr.25244) 232 : cluster [DBG] pgmap v172: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:02:23.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:02:22 smithi089 bash[82202]: cluster 2024-07-15T07:02:22.825916+0000 mgr.y (mgr.25244) 232 : cluster [DBG] pgmap v172: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:02:24.620 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 07:02:24 smithi089 bash[72894]: ::ffff:172.21.15.134 - - [15/Jul/2024:07:02:24] "GET /metrics HTTP/1.1" 200 33984 "" "Prometheus/2.43.0" 2024-07-15T07:02:25.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:02:24 smithi134 bash[67624]: cluster 2024-07-15T07:02:24.826631+0000 mgr.y (mgr.25244) 233 : cluster [DBG] pgmap v173: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:02:25.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:02:24 smithi089 bash[79810]: cluster 2024-07-15T07:02:24.826631+0000 mgr.y (mgr.25244) 233 : cluster [DBG] pgmap v173: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:02:25.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:02:24 smithi089 bash[82202]: cluster 2024-07-15T07:02:24.826631+0000 mgr.y (mgr.25244) 233 : cluster [DBG] pgmap v173: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:02:27.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:02:26 smithi134 bash[67624]: cluster 2024-07-15T07:02:26.827660+0000 mgr.y (mgr.25244) 234 : cluster [DBG] pgmap v174: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:02:27.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:02:26 smithi089 bash[79810]: cluster 2024-07-15T07:02:26.827660+0000 mgr.y (mgr.25244) 234 : cluster [DBG] pgmap v174: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:02:27.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:02:26 smithi089 bash[82202]: cluster 2024-07-15T07:02:26.827660+0000 mgr.y (mgr.25244) 234 : cluster [DBG] pgmap v174: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:02:29.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:02:28 smithi134 bash[67624]: cluster 2024-07-15T07:02:28.828487+0000 mgr.y (mgr.25244) 235 : cluster [DBG] pgmap v175: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:02:29.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:02:28 smithi089 bash[79810]: cluster 2024-07-15T07:02:28.828487+0000 mgr.y (mgr.25244) 235 : cluster [DBG] pgmap v175: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:02:29.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:02:28 smithi089 bash[82202]: cluster 2024-07-15T07:02:28.828487+0000 mgr.y (mgr.25244) 235 : cluster [DBG] pgmap v175: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:02:31.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:02:30 smithi134 bash[67624]: cluster 2024-07-15T07:02:30.829449+0000 mgr.y (mgr.25244) 236 : cluster [DBG] pgmap v176: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:02:31.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:02:30 smithi089 bash[79810]: cluster 2024-07-15T07:02:30.829449+0000 mgr.y (mgr.25244) 236 : cluster [DBG] pgmap v176: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:02:31.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:02:30 smithi089 bash[82202]: cluster 2024-07-15T07:02:30.829449+0000 mgr.y (mgr.25244) 236 : cluster [DBG] pgmap v176: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:02:32.804 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:02:32 smithi134 bash[60729]: ts=2024-07-15T07:02:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T07:02:33.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:02:32 smithi134 bash[67624]: cluster 2024-07-15T07:02:32.830093+0000 mgr.y (mgr.25244) 237 : cluster [DBG] pgmap v177: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:02:33.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:02:32 smithi089 bash[79810]: cluster 2024-07-15T07:02:32.830093+0000 mgr.y (mgr.25244) 237 : cluster [DBG] pgmap v177: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:02:33.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:02:32 smithi089 bash[82202]: cluster 2024-07-15T07:02:32.830093+0000 mgr.y (mgr.25244) 237 : cluster [DBG] pgmap v177: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:02:34.620 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 07:02:34 smithi089 bash[72894]: ::ffff:172.21.15.134 - - [15/Jul/2024:07:02:34] "GET /metrics HTTP/1.1" 200 33984 "" "Prometheus/2.43.0" 2024-07-15T07:02:35.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:02:34 smithi089 bash[79810]: audit 2024-07-15T07:02:34.791734+0000 mon.a (mon.0) 376 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:02:35.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:02:34 smithi089 bash[79810]: cluster 2024-07-15T07:02:34.830758+0000 mgr.y (mgr.25244) 238 : cluster [DBG] pgmap v178: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:02:35.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:02:34 smithi089 bash[82202]: audit 2024-07-15T07:02:34.791734+0000 mon.a (mon.0) 376 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:02:35.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:02:34 smithi089 bash[82202]: cluster 2024-07-15T07:02:34.830758+0000 mgr.y (mgr.25244) 238 : cluster [DBG] pgmap v178: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:02:35.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:02:34 smithi134 bash[67624]: audit 2024-07-15T07:02:34.791734+0000 mon.a (mon.0) 376 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:02:35.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:02:34 smithi134 bash[67624]: cluster 2024-07-15T07:02:34.830758+0000 mgr.y (mgr.25244) 238 : cluster [DBG] pgmap v178: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:02:37.224 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:02:36 smithi089 bash[82202]: cluster 2024-07-15T07:02:36.832167+0000 mgr.y (mgr.25244) 239 : cluster [DBG] pgmap v179: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:02:37.224 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:02:36 smithi089 bash[79810]: cluster 2024-07-15T07:02:36.832167+0000 mgr.y (mgr.25244) 239 : cluster [DBG] pgmap v179: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:02:37.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:02:36 smithi134 bash[67624]: cluster 2024-07-15T07:02:36.832167+0000 mgr.y (mgr.25244) 239 : cluster [DBG] pgmap v179: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:02:37.992 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -e sha1=6ea7aaa9f93d34dc4d21862afe00140dfddbd706 -- bash -c 'ceph orch upgrade resume' 2024-07-15T07:02:39.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:02:38 smithi134 bash[67624]: cluster 2024-07-15T07:02:38.832911+0000 mgr.y (mgr.25244) 240 : cluster [DBG] pgmap v180: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:02:39.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:02:38 smithi089 bash[79810]: cluster 2024-07-15T07:02:38.832911+0000 mgr.y (mgr.25244) 240 : cluster [DBG] pgmap v180: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:02:39.371 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:02:38 smithi089 bash[82202]: cluster 2024-07-15T07:02:38.832911+0000 mgr.y (mgr.25244) 240 : cluster [DBG] pgmap v180: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:02:41.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:02:40 smithi134 bash[67624]: cluster 2024-07-15T07:02:40.833849+0000 mgr.y (mgr.25244) 241 : cluster [DBG] pgmap v181: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:02:41.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:02:40 smithi089 bash[79810]: cluster 2024-07-15T07:02:40.833849+0000 mgr.y (mgr.25244) 241 : cluster [DBG] pgmap v181: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:02:41.371 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:02:40 smithi089 bash[82202]: cluster 2024-07-15T07:02:40.833849+0000 mgr.y (mgr.25244) 241 : cluster [DBG] pgmap v181: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:02:42.775 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T07:02:42.804 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:02:42 smithi134 bash[60729]: ts=2024-07-15T07:02:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T07:02:43.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:02:42 smithi134 bash[67624]: cluster 2024-07-15T07:02:42.834443+0000 mgr.y (mgr.25244) 242 : cluster [DBG] pgmap v182: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:02:43.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:02:42 smithi089 bash[79810]: cluster 2024-07-15T07:02:42.834443+0000 mgr.y (mgr.25244) 242 : cluster [DBG] pgmap v182: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:02:43.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:02:42 smithi089 bash[82202]: cluster 2024-07-15T07:02:42.834443+0000 mgr.y (mgr.25244) 242 : cluster [DBG] pgmap v182: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:02:44.353 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 07:02:44 smithi089 bash[72894]: ::ffff:172.21.15.134 - - [15/Jul/2024:07:02:44] "GET /metrics HTTP/1.1" 200 33984 "" "Prometheus/2.43.0" 2024-07-15T07:02:45.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:02:44 smithi134 bash[67624]: cluster 2024-07-15T07:02:44.835199+0000 mgr.y (mgr.25244) 243 : cluster [DBG] pgmap v183: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:02:45.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:02:44 smithi089 bash[79810]: cluster 2024-07-15T07:02:44.835199+0000 mgr.y (mgr.25244) 243 : cluster [DBG] pgmap v183: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:02:45.371 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:02:44 smithi089 bash[82202]: cluster 2024-07-15T07:02:44.835199+0000 mgr.y (mgr.25244) 243 : cluster [DBG] pgmap v183: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:02:46.389 INFO:teuthology.orchestra.run.smithi089.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:badc66b92d2a2cfbbce7f293b5accdc50f486d2b2d966876e46fe28d05f464f1 2024-07-15T07:02:47.102 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -e sha1=6ea7aaa9f93d34dc4d21862afe00140dfddbd706 -- bash -c 'echo wait for half of osds to upgrade' 2024-07-15T07:02:47.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:02:47 smithi134 bash[67624]: audit 2024-07-15T07:02:46.387908+0000 mgr.y (mgr.25244) 244 : audit [DBG] from='client.44156 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T07:02:47.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:02:47 smithi134 bash[67624]: cephadm 2024-07-15T07:02:46.388282+0000 mgr.y (mgr.25244) 245 : cephadm [INF] Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:badc66b92d2a2cfbbce7f293b5accdc50f486d2b2d966876e46fe28d05f464f1 2024-07-15T07:02:47.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:02:47 smithi134 bash[67624]: audit 2024-07-15T07:02:46.393059+0000 mon.a (mon.0) 377 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:02:47.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:02:47 smithi134 bash[67624]: audit 2024-07-15T07:02:46.393844+0000 mon.a (mon.0) 378 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T07:02:47.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:02:47 smithi134 bash[67624]: cluster 2024-07-15T07:02:46.836204+0000 mgr.y (mgr.25244) 246 : cluster [DBG] pgmap v184: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:02:47.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:02:47 smithi089 bash[79810]: audit 2024-07-15T07:02:46.387908+0000 mgr.y (mgr.25244) 244 : audit [DBG] from='client.44156 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T07:02:47.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:02:47 smithi089 bash[79810]: cephadm 2024-07-15T07:02:46.388282+0000 mgr.y (mgr.25244) 245 : cephadm [INF] Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:badc66b92d2a2cfbbce7f293b5accdc50f486d2b2d966876e46fe28d05f464f1 2024-07-15T07:02:47.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:02:47 smithi089 bash[79810]: audit 2024-07-15T07:02:46.393059+0000 mon.a (mon.0) 377 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:02:47.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:02:47 smithi089 bash[79810]: audit 2024-07-15T07:02:46.393844+0000 mon.a (mon.0) 378 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T07:02:47.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:02:47 smithi089 bash[79810]: cluster 2024-07-15T07:02:46.836204+0000 mgr.y (mgr.25244) 246 : cluster [DBG] pgmap v184: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:02:47.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:02:47 smithi089 bash[82202]: audit 2024-07-15T07:02:46.387908+0000 mgr.y (mgr.25244) 244 : audit [DBG] from='client.44156 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T07:02:47.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:02:47 smithi089 bash[82202]: cephadm 2024-07-15T07:02:46.388282+0000 mgr.y (mgr.25244) 245 : cephadm [INF] Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:badc66b92d2a2cfbbce7f293b5accdc50f486d2b2d966876e46fe28d05f464f1 2024-07-15T07:02:47.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:02:47 smithi089 bash[82202]: audit 2024-07-15T07:02:46.393059+0000 mon.a (mon.0) 377 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:02:47.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:02:47 smithi089 bash[82202]: audit 2024-07-15T07:02:46.393844+0000 mon.a (mon.0) 378 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T07:02:47.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:02:47 smithi089 bash[82202]: cluster 2024-07-15T07:02:46.836204+0000 mgr.y (mgr.25244) 246 : cluster [DBG] pgmap v184: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:02:49.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:02:48 smithi134 bash[67624]: cluster 2024-07-15T07:02:48.837022+0000 mgr.y (mgr.25244) 247 : cluster [DBG] pgmap v185: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:02:49.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:02:48 smithi089 bash[79810]: cluster 2024-07-15T07:02:48.837022+0000 mgr.y (mgr.25244) 247 : cluster [DBG] pgmap v185: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:02:49.371 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:02:48 smithi089 bash[82202]: cluster 2024-07-15T07:02:48.837022+0000 mgr.y (mgr.25244) 247 : cluster [DBG] pgmap v185: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:02:50.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:02:49 smithi134 bash[67624]: audit 2024-07-15T07:02:49.792368+0000 mon.a (mon.0) 379 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:02:50.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:02:49 smithi089 bash[79810]: audit 2024-07-15T07:02:49.792368+0000 mon.a (mon.0) 379 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:02:50.371 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:02:49 smithi089 bash[82202]: audit 2024-07-15T07:02:49.792368+0000 mon.a (mon.0) 379 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:02:51.261 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T07:02:51.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:02:50 smithi134 bash[67624]: cluster 2024-07-15T07:02:50.838000+0000 mgr.y (mgr.25244) 248 : cluster [DBG] pgmap v186: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:02:51.352 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:02:50 smithi089 bash[79810]: cluster 2024-07-15T07:02:50.838000+0000 mgr.y (mgr.25244) 248 : cluster [DBG] pgmap v186: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:02:51.353 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:02:50 smithi089 bash[82202]: cluster 2024-07-15T07:02:50.838000+0000 mgr.y (mgr.25244) 248 : cluster [DBG] pgmap v186: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:02:52.805 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:02:52 smithi134 bash[60729]: ts=2024-07-15T07:02:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T07:02:53.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:02:52 smithi089 bash[79810]: audit 2024-07-15T07:02:51.929600+0000 mon.a (mon.0) 380 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:02:53.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:02:52 smithi089 bash[79810]: audit 2024-07-15T07:02:51.935768+0000 mon.a (mon.0) 381 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:02:53.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:02:52 smithi089 bash[79810]: audit 2024-07-15T07:02:52.367008+0000 mon.a (mon.0) 382 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T07:02:53.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:02:52 smithi089 bash[79810]: audit 2024-07-15T07:02:52.368707+0000 mon.a (mon.0) 383 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T07:02:53.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:02:52 smithi089 bash[79810]: audit 2024-07-15T07:02:52.378465+0000 mon.a (mon.0) 384 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:02:53.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:02:52 smithi089 bash[79810]: audit 2024-07-15T07:02:52.512560+0000 mon.a (mon.0) 385 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T07:02:53.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:02:52 smithi089 bash[79810]: audit 2024-07-15T07:02:52.516902+0000 mon.a (mon.0) 386 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-15T07:02:53.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:02:52 smithi089 bash[79810]: audit 2024-07-15T07:02:52.519457+0000 mon.a (mon.0) 387 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-15T07:02:53.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:02:52 smithi089 bash[79810]: audit 2024-07-15T07:02:52.521504+0000 mon.a (mon.0) 388 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-15T07:02:53.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:02:52 smithi089 bash[79810]: audit 2024-07-15T07:02:52.523894+0000 mon.a (mon.0) 389 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-07-15T07:02:53.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:02:52 smithi089 bash[79810]: audit 2024-07-15T07:02:52.524144+0000 mgr.y (mgr.25244) 249 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-07-15T07:02:53.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:02:52 smithi089 bash[79810]: cephadm 2024-07-15T07:02:52.525835+0000 mgr.y (mgr.25244) 250 : cephadm [INF] Upgrade: osd.2 is safe to restart 2024-07-15T07:02:53.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:02:52 smithi089 bash[79810]: cluster 2024-07-15T07:02:52.838604+0000 mgr.y (mgr.25244) 251 : cluster [DBG] pgmap v187: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:02:53.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:02:52 smithi089 bash[82202]: audit 2024-07-15T07:02:51.929600+0000 mon.a (mon.0) 380 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:02:53.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:02:52 smithi089 bash[82202]: audit 2024-07-15T07:02:51.935768+0000 mon.a (mon.0) 381 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:02:53.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:02:52 smithi089 bash[82202]: audit 2024-07-15T07:02:52.367008+0000 mon.a (mon.0) 382 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T07:02:53.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:02:52 smithi089 bash[82202]: audit 2024-07-15T07:02:52.368707+0000 mon.a (mon.0) 383 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T07:02:53.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:02:52 smithi089 bash[82202]: audit 2024-07-15T07:02:52.378465+0000 mon.a (mon.0) 384 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:02:53.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:02:52 smithi089 bash[82202]: audit 2024-07-15T07:02:52.512560+0000 mon.a (mon.0) 385 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T07:02:53.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:02:52 smithi089 bash[82202]: audit 2024-07-15T07:02:52.516902+0000 mon.a (mon.0) 386 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-15T07:02:53.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:02:52 smithi089 bash[82202]: audit 2024-07-15T07:02:52.519457+0000 mon.a (mon.0) 387 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-15T07:02:53.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:02:52 smithi089 bash[82202]: audit 2024-07-15T07:02:52.521504+0000 mon.a (mon.0) 388 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-15T07:02:53.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:02:52 smithi089 bash[82202]: audit 2024-07-15T07:02:52.523894+0000 mon.a (mon.0) 389 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-07-15T07:02:53.123 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:02:52 smithi089 bash[82202]: audit 2024-07-15T07:02:52.524144+0000 mgr.y (mgr.25244) 249 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-07-15T07:02:53.123 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:02:52 smithi089 bash[82202]: cephadm 2024-07-15T07:02:52.525835+0000 mgr.y (mgr.25244) 250 : cephadm [INF] Upgrade: osd.2 is safe to restart 2024-07-15T07:02:53.123 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:02:52 smithi089 bash[82202]: cluster 2024-07-15T07:02:52.838604+0000 mgr.y (mgr.25244) 251 : cluster [DBG] pgmap v187: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:02:53.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:02:52 smithi134 bash[67624]: audit 2024-07-15T07:02:51.929600+0000 mon.a (mon.0) 380 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:02:53.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:02:52 smithi134 bash[67624]: audit 2024-07-15T07:02:51.935768+0000 mon.a (mon.0) 381 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:02:53.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:02:52 smithi134 bash[67624]: audit 2024-07-15T07:02:52.367008+0000 mon.a (mon.0) 382 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T07:02:53.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:02:52 smithi134 bash[67624]: audit 2024-07-15T07:02:52.368707+0000 mon.a (mon.0) 383 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T07:02:53.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:02:52 smithi134 bash[67624]: audit 2024-07-15T07:02:52.378465+0000 mon.a (mon.0) 384 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:02:53.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:02:52 smithi134 bash[67624]: audit 2024-07-15T07:02:52.512560+0000 mon.a (mon.0) 385 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T07:02:53.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:02:52 smithi134 bash[67624]: audit 2024-07-15T07:02:52.516902+0000 mon.a (mon.0) 386 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-15T07:02:53.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:02:52 smithi134 bash[67624]: audit 2024-07-15T07:02:52.519457+0000 mon.a (mon.0) 387 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-15T07:02:53.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:02:52 smithi134 bash[67624]: audit 2024-07-15T07:02:52.521504+0000 mon.a (mon.0) 388 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-15T07:02:53.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:02:52 smithi134 bash[67624]: audit 2024-07-15T07:02:52.523894+0000 mon.a (mon.0) 389 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-07-15T07:02:53.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:02:52 smithi134 bash[67624]: audit 2024-07-15T07:02:52.524144+0000 mgr.y (mgr.25244) 249 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-07-15T07:02:53.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:02:52 smithi134 bash[67624]: cephadm 2024-07-15T07:02:52.525835+0000 mgr.y (mgr.25244) 250 : cephadm [INF] Upgrade: osd.2 is safe to restart 2024-07-15T07:02:53.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:02:52 smithi134 bash[67624]: cluster 2024-07-15T07:02:52.838604+0000 mgr.y (mgr.25244) 251 : cluster [DBG] pgmap v187: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:02:54.735 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 07:02:54 smithi089 bash[72894]: ::ffff:172.21.15.134 - - [15/Jul/2024:07:02:54] "GET /metrics HTTP/1.1" 200 33984 "" "Prometheus/2.43.0" 2024-07-15T07:02:55.150 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:02:54 smithi089 bash[82202]: cluster 2024-07-15T07:02:54.839333+0000 mgr.y (mgr.25244) 252 : cluster [DBG] pgmap v188: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:02:55.150 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:02:54 smithi089 bash[79810]: cluster 2024-07-15T07:02:54.839333+0000 mgr.y (mgr.25244) 252 : cluster [DBG] pgmap v188: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:02:55.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:02:54 smithi134 bash[67624]: cluster 2024-07-15T07:02:54.839333+0000 mgr.y (mgr.25244) 252 : cluster [DBG] pgmap v188: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:02:55.539 INFO:teuthology.orchestra.run.smithi089.stdout:wait for half of osds to upgrade 2024-07-15T07:02:56.695 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -e sha1=6ea7aaa9f93d34dc4d21862afe00140dfddbd706 -- bash -c 'while ! ceph osd versions | grep $sha1 | egrep '"'"': [45678]'"'"'; do sleep 2 ; done' 2024-07-15T07:02:57.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:02:57 smithi134 bash[67624]: cephadm 2024-07-15T07:02:56.516051+0000 mgr.y (mgr.25244) 253 : cephadm [INF] Upgrade: Updating osd.2 2024-07-15T07:02:57.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:02:57 smithi134 bash[67624]: audit 2024-07-15T07:02:56.524994+0000 mon.a (mon.0) 390 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:02:57.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:02:57 smithi134 bash[67624]: audit 2024-07-15T07:02:56.526187+0000 mon.a (mon.0) 391 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-07-15T07:02:57.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:02:57 smithi134 bash[67624]: audit 2024-07-15T07:02:56.527049+0000 mon.a (mon.0) 392 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T07:02:57.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:02:57 smithi134 bash[67624]: cephadm 2024-07-15T07:02:56.530153+0000 mgr.y (mgr.25244) 254 : cephadm [INF] Deploying daemon osd.2 on smithi089 2024-07-15T07:02:57.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:02:57 smithi134 bash[67624]: cluster 2024-07-15T07:02:56.839978+0000 mgr.y (mgr.25244) 255 : cluster [DBG] pgmap v189: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:02:57.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:02:57 smithi089 bash[79810]: cephadm 2024-07-15T07:02:56.516051+0000 mgr.y (mgr.25244) 253 : cephadm [INF] Upgrade: Updating osd.2 2024-07-15T07:02:57.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:02:57 smithi089 bash[79810]: audit 2024-07-15T07:02:56.524994+0000 mon.a (mon.0) 390 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:02:57.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:02:57 smithi089 bash[79810]: audit 2024-07-15T07:02:56.526187+0000 mon.a (mon.0) 391 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-07-15T07:02:57.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:02:57 smithi089 bash[79810]: audit 2024-07-15T07:02:56.527049+0000 mon.a (mon.0) 392 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T07:02:57.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:02:57 smithi089 bash[79810]: cephadm 2024-07-15T07:02:56.530153+0000 mgr.y (mgr.25244) 254 : cephadm [INF] Deploying daemon osd.2 on smithi089 2024-07-15T07:02:57.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:02:57 smithi089 bash[79810]: cluster 2024-07-15T07:02:56.839978+0000 mgr.y (mgr.25244) 255 : cluster [DBG] pgmap v189: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:02:57.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:02:57 smithi089 bash[82202]: cephadm 2024-07-15T07:02:56.516051+0000 mgr.y (mgr.25244) 253 : cephadm [INF] Upgrade: Updating osd.2 2024-07-15T07:02:57.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:02:57 smithi089 bash[82202]: audit 2024-07-15T07:02:56.524994+0000 mon.a (mon.0) 390 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:02:57.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:02:57 smithi089 bash[82202]: audit 2024-07-15T07:02:56.526187+0000 mon.a (mon.0) 391 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-07-15T07:02:57.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:02:57 smithi089 bash[82202]: audit 2024-07-15T07:02:56.527049+0000 mon.a (mon.0) 392 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T07:02:57.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:02:57 smithi089 bash[82202]: cephadm 2024-07-15T07:02:56.530153+0000 mgr.y (mgr.25244) 254 : cephadm [INF] Deploying daemon osd.2 on smithi089 2024-07-15T07:02:57.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:02:57 smithi089 bash[82202]: cluster 2024-07-15T07:02:56.839978+0000 mgr.y (mgr.25244) 255 : cluster [DBG] pgmap v189: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:02:59.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:02:58 smithi134 bash[67624]: cluster 2024-07-15T07:02:58.840435+0000 mgr.y (mgr.25244) 256 : cluster [DBG] pgmap v190: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:02:59.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:02:58 smithi089 bash[79810]: cluster 2024-07-15T07:02:58.840435+0000 mgr.y (mgr.25244) 256 : cluster [DBG] pgmap v190: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:02:59.371 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:02:58 smithi089 bash[82202]: cluster 2024-07-15T07:02:58.840435+0000 mgr.y (mgr.25244) 256 : cluster [DBG] pgmap v190: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:03:01.160 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:00 smithi089 bash[82202]: cluster 2024-07-15T07:03:00.841422+0000 mgr.y (mgr.25244) 257 : cluster [DBG] pgmap v191: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:03:01.160 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:00 smithi089 bash[79810]: cluster 2024-07-15T07:03:00.841422+0000 mgr.y (mgr.25244) 257 : cluster [DBG] pgmap v191: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:03:01.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:00 smithi134 bash[67624]: cluster 2024-07-15T07:03:00.841422+0000 mgr.y (mgr.25244) 257 : cluster [DBG] pgmap v191: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:03:01.431 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:01 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:03:01.432 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 07:03:01 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:03:01.432 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:01 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:03:01.432 INFO:journalctl@ceph.osd.3.smithi089.stdout:Jul 15 07:03:01 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:03:01.433 INFO:journalctl@ceph.osd.0.smithi089.stdout:Jul 15 07:03:01 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:03:01.433 INFO:journalctl@ceph.osd.2.smithi089.stdout:Jul 15 07:03:01 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:03:01.433 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 07:03:01 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:03:01.434 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 07:03:01 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:03:01.434 INFO:journalctl@ceph.osd.1.smithi089.stdout:Jul 15 07:03:01 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:03:01.471 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T07:03:01.870 INFO:journalctl@ceph.osd.2.smithi089.stdout:Jul 15 07:03:01 smithi089 systemd[1]: Stopping Ceph osd.2 for 75ba02bc-4274-11ef-bcad-c7b262605968... 2024-07-15T07:03:01.871 INFO:journalctl@ceph.osd.2.smithi089.stdout:Jul 15 07:03:01 smithi089 bash[45524]: debug 2024-07-15T07:03:01.582+0000 7f5d4e983640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-07-15T07:03:01.871 INFO:journalctl@ceph.osd.2.smithi089.stdout:Jul 15 07:03:01 smithi089 bash[45524]: debug 2024-07-15T07:03:01.582+0000 7f5d4e983640 -1 osd.2 89 *** Got signal Terminated *** 2024-07-15T07:03:01.871 INFO:journalctl@ceph.osd.2.smithi089.stdout:Jul 15 07:03:01 smithi089 bash[45524]: debug 2024-07-15T07:03:01.582+0000 7f5d4e983640 -1 osd.2 89 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-07-15T07:03:02.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:01 smithi134 bash[67624]: cluster 2024-07-15T07:03:01.585711+0000 mon.a (mon.0) 393 : cluster [INF] osd.2 marked itself down and dead 2024-07-15T07:03:02.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:01 smithi089 bash[82202]: cluster 2024-07-15T07:03:01.585711+0000 mon.a (mon.0) 393 : cluster [INF] osd.2 marked itself down and dead 2024-07-15T07:03:02.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:01 smithi089 bash[79810]: debug 2024-07-15T07:03:01.898+0000 7fb563a0a640 -1 log_channel(cluster) log [ERR] : Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-07-15T07:03:02.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:01 smithi089 bash[79810]: cluster 2024-07-15T07:03:01.585711+0000 mon.a (mon.0) 393 : cluster [INF] osd.2 marked itself down and dead 2024-07-15T07:03:02.804 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:03:02 smithi134 bash[60729]: ts=2024-07-15T07:03:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T07:03:03.061 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:02 smithi089 bash[82202]: cluster 2024-07-15T07:03:01.902928+0000 mon.a (mon.0) 394 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-07-15T07:03:03.061 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:02 smithi089 bash[82202]: cluster 2024-07-15T07:03:01.902975+0000 mon.a (mon.0) 395 : cluster [ERR] Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-07-15T07:03:03.061 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:02 smithi089 bash[82202]: cluster 2024-07-15T07:03:01.920305+0000 mon.a (mon.0) 396 : cluster [DBG] osdmap e90: 8 total, 7 up, 8 in 2024-07-15T07:03:03.061 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:02 smithi089 bash[82202]: cluster 2024-07-15T07:03:02.841994+0000 mgr.y (mgr.25244) 258 : cluster [DBG] pgmap v193: 73 pgs: 2 stale+active+clean, 71 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:03:03.062 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:02 smithi089 bash[79810]: cluster 2024-07-15T07:03:01.902928+0000 mon.a (mon.0) 394 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-07-15T07:03:03.062 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:02 smithi089 bash[79810]: cluster 2024-07-15T07:03:01.902975+0000 mon.a (mon.0) 395 : cluster [ERR] Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-07-15T07:03:03.062 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:02 smithi089 bash[79810]: cluster 2024-07-15T07:03:01.920305+0000 mon.a (mon.0) 396 : cluster [DBG] osdmap e90: 8 total, 7 up, 8 in 2024-07-15T07:03:03.062 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:02 smithi089 bash[79810]: cluster 2024-07-15T07:03:02.841994+0000 mgr.y (mgr.25244) 258 : cluster [DBG] pgmap v193: 73 pgs: 2 stale+active+clean, 71 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:03:03.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:02 smithi134 bash[67624]: cluster 2024-07-15T07:03:01.902928+0000 mon.a (mon.0) 394 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-07-15T07:03:03.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:02 smithi134 bash[67624]: cluster 2024-07-15T07:03:01.902975+0000 mon.a (mon.0) 395 : cluster [ERR] Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-07-15T07:03:03.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:02 smithi134 bash[67624]: cluster 2024-07-15T07:03:01.920305+0000 mon.a (mon.0) 396 : cluster [DBG] osdmap e90: 8 total, 7 up, 8 in 2024-07-15T07:03:03.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:02 smithi134 bash[67624]: cluster 2024-07-15T07:03:02.841994+0000 mgr.y (mgr.25244) 258 : cluster [DBG] pgmap v193: 73 pgs: 2 stale+active+clean, 71 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-15T07:03:03.370 INFO:journalctl@ceph.osd.2.smithi089.stdout:Jul 15 07:03:03 smithi089 bash[97802]: ceph-75ba02bc-4274-11ef-bcad-c7b262605968-osd-2 2024-07-15T07:03:03.716 INFO:journalctl@ceph.osd.2.smithi089.stdout:Jul 15 07:03:03 smithi089 bash[97874]: Error response from daemon: No such container: ceph-75ba02bc-4274-11ef-bcad-c7b262605968-osd-2 2024-07-15T07:03:04.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:03 smithi089 bash[82202]: cluster 2024-07-15T07:03:02.923110+0000 mon.a (mon.0) 397 : cluster [DBG] osdmap e91: 8 total, 7 up, 8 in 2024-07-15T07:03:04.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:03 smithi089 bash[79810]: cluster 2024-07-15T07:03:02.923110+0000 mon.a (mon.0) 397 : cluster [DBG] osdmap e91: 8 total, 7 up, 8 in 2024-07-15T07:03:04.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:03 smithi134 bash[67624]: cluster 2024-07-15T07:03:02.923110+0000 mon.a (mon.0) 397 : cluster [DBG] osdmap e91: 8 total, 7 up, 8 in 2024-07-15T07:03:04.620 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 07:03:04 smithi089 bash[72894]: ::ffff:172.21.15.134 - - [15/Jul/2024:07:03:04] "GET /metrics HTTP/1.1" 200 33984 "" "Prometheus/2.43.0" 2024-07-15T07:03:05.177 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:04 smithi089 bash[79810]: audit 2024-07-15T07:03:04.755573+0000 mon.a (mon.0) 398 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2024-07-15T07:03:05.177 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:04 smithi089 bash[79810]: audit 2024-07-15T07:03:04.756081+0000 mon.a (mon.0) 399 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.c", "id": [4, 2]}]: dispatch 2024-07-15T07:03:05.177 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:04 smithi089 bash[79810]: audit 2024-07-15T07:03:04.756474+0000 mon.a (mon.0) 400 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 0]}]: dispatch 2024-07-15T07:03:05.177 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:04 smithi089 bash[79810]: audit 2024-07-15T07:03:04.756840+0000 mon.a (mon.0) 401 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2024-07-15T07:03:05.177 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:04 smithi089 bash[79810]: audit 2024-07-15T07:03:04.757208+0000 mon.a (mon.0) 402 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [3, 2]}]: dispatch 2024-07-15T07:03:05.177 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:04 smithi089 bash[79810]: audit 2024-07-15T07:03:04.757592+0000 mon.a (mon.0) 403 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 0, 3, 2]}]: dispatch 2024-07-15T07:03:05.177 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:04 smithi089 bash[79810]: audit 2024-07-15T07:03:04.757979+0000 mon.a (mon.0) 404 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [3, 5, 4, 2]}]: dispatch 2024-07-15T07:03:05.177 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:04 smithi089 bash[79810]: audit 2024-07-15T07:03:04.758362+0000 mon.a (mon.0) 405 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1e", "id": [3, 2]}]: dispatch 2024-07-15T07:03:05.177 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:04 smithi089 bash[79810]: audit 2024-07-15T07:03:04.797323+0000 mon.a (mon.0) 406 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:03:05.177 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:04 smithi089 bash[79810]: audit 2024-07-15T07:03:04.799145+0000 mon.a (mon.0) 407 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:03:05.177 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:04 smithi089 bash[79810]: cluster 2024-07-15T07:03:04.842715+0000 mgr.y (mgr.25244) 259 : cluster [DBG] pgmap v195: 73 pgs: 1 active+undersized, 2 stale+active+clean, 1 active+undersized+degraded, 69 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2024-07-15T07:03:05.178 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:04 smithi089 bash[82202]: audit 2024-07-15T07:03:04.755573+0000 mon.a (mon.0) 398 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2024-07-15T07:03:05.178 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:04 smithi089 bash[82202]: audit 2024-07-15T07:03:04.756081+0000 mon.a (mon.0) 399 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.c", "id": [4, 2]}]: dispatch 2024-07-15T07:03:05.178 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:04 smithi089 bash[82202]: audit 2024-07-15T07:03:04.756474+0000 mon.a (mon.0) 400 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 0]}]: dispatch 2024-07-15T07:03:05.178 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:04 smithi089 bash[82202]: audit 2024-07-15T07:03:04.756840+0000 mon.a (mon.0) 401 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2024-07-15T07:03:05.178 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:04 smithi089 bash[82202]: audit 2024-07-15T07:03:04.757208+0000 mon.a (mon.0) 402 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [3, 2]}]: dispatch 2024-07-15T07:03:05.178 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:04 smithi089 bash[82202]: audit 2024-07-15T07:03:04.757592+0000 mon.a (mon.0) 403 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 0, 3, 2]}]: dispatch 2024-07-15T07:03:05.178 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:04 smithi089 bash[82202]: audit 2024-07-15T07:03:04.757979+0000 mon.a (mon.0) 404 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [3, 5, 4, 2]}]: dispatch 2024-07-15T07:03:05.178 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:04 smithi089 bash[82202]: audit 2024-07-15T07:03:04.758362+0000 mon.a (mon.0) 405 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1e", "id": [3, 2]}]: dispatch 2024-07-15T07:03:05.178 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:04 smithi089 bash[82202]: audit 2024-07-15T07:03:04.797323+0000 mon.a (mon.0) 406 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:03:05.179 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:04 smithi089 bash[82202]: audit 2024-07-15T07:03:04.799145+0000 mon.a (mon.0) 407 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:03:05.179 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:04 smithi089 bash[82202]: cluster 2024-07-15T07:03:04.842715+0000 mgr.y (mgr.25244) 259 : cluster [DBG] pgmap v195: 73 pgs: 1 active+undersized, 2 stale+active+clean, 1 active+undersized+degraded, 69 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2024-07-15T07:03:05.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:04 smithi134 bash[67624]: audit 2024-07-15T07:03:04.755573+0000 mon.a (mon.0) 398 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2024-07-15T07:03:05.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:04 smithi134 bash[67624]: audit 2024-07-15T07:03:04.756081+0000 mon.a (mon.0) 399 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.c", "id": [4, 2]}]: dispatch 2024-07-15T07:03:05.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:04 smithi134 bash[67624]: audit 2024-07-15T07:03:04.756474+0000 mon.a (mon.0) 400 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 0]}]: dispatch 2024-07-15T07:03:05.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:04 smithi134 bash[67624]: audit 2024-07-15T07:03:04.756840+0000 mon.a (mon.0) 401 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2024-07-15T07:03:05.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:04 smithi134 bash[67624]: audit 2024-07-15T07:03:04.757208+0000 mon.a (mon.0) 402 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [3, 2]}]: dispatch 2024-07-15T07:03:05.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:04 smithi134 bash[67624]: audit 2024-07-15T07:03:04.757592+0000 mon.a (mon.0) 403 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 0, 3, 2]}]: dispatch 2024-07-15T07:03:05.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:04 smithi134 bash[67624]: audit 2024-07-15T07:03:04.757979+0000 mon.a (mon.0) 404 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [3, 5, 4, 2]}]: dispatch 2024-07-15T07:03:05.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:04 smithi134 bash[67624]: audit 2024-07-15T07:03:04.758362+0000 mon.a (mon.0) 405 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1e", "id": [3, 2]}]: dispatch 2024-07-15T07:03:05.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:04 smithi134 bash[67624]: audit 2024-07-15T07:03:04.797323+0000 mon.a (mon.0) 406 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:03:05.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:04 smithi134 bash[67624]: audit 2024-07-15T07:03:04.799145+0000 mon.a (mon.0) 407 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:03:05.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:04 smithi134 bash[67624]: cluster 2024-07-15T07:03:04.842715+0000 mgr.y (mgr.25244) 259 : cluster [DBG] pgmap v195: 73 pgs: 1 active+undersized, 2 stale+active+clean, 1 active+undersized+degraded, 69 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2024-07-15T07:03:06.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:05 smithi089 bash[79810]: audit 2024-07-15T07:03:04.942979+0000 mon.a (mon.0) 408 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2024-07-15T07:03:06.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:05 smithi089 bash[79810]: audit 2024-07-15T07:03:04.943132+0000 mon.a (mon.0) 409 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.c", "id": [4, 2]}]': finished 2024-07-15T07:03:06.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:05 smithi089 bash[79810]: audit 2024-07-15T07:03:04.943234+0000 mon.a (mon.0) 410 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 0]}]': finished 2024-07-15T07:03:06.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:05 smithi089 bash[79810]: audit 2024-07-15T07:03:04.943372+0000 mon.a (mon.0) 411 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]': finished 2024-07-15T07:03:06.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:05 smithi089 bash[79810]: audit 2024-07-15T07:03:04.943494+0000 mon.a (mon.0) 412 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [3, 2]}]': finished 2024-07-15T07:03:06.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:05 smithi089 bash[79810]: audit 2024-07-15T07:03:04.943663+0000 mon.a (mon.0) 413 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 0, 3, 2]}]': finished 2024-07-15T07:03:06.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:05 smithi089 bash[79810]: audit 2024-07-15T07:03:04.943849+0000 mon.a (mon.0) 414 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [3, 5, 4, 2]}]': finished 2024-07-15T07:03:06.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:05 smithi089 bash[79810]: audit 2024-07-15T07:03:04.944054+0000 mon.a (mon.0) 415 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1e", "id": [3, 2]}]': finished 2024-07-15T07:03:06.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:05 smithi089 bash[79810]: cluster 2024-07-15T07:03:04.944998+0000 mon.a (mon.0) 416 : cluster [DBG] osdmap e92: 8 total, 7 up, 8 in 2024-07-15T07:03:06.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:05 smithi089 bash[79810]: cluster 2024-07-15T07:03:05.794442+0000 mon.a (mon.0) 417 : cluster [WRN] Health check failed: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded (PG_DEGRADED) 2024-07-15T07:03:06.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:05 smithi089 bash[82202]: audit 2024-07-15T07:03:04.942979+0000 mon.a (mon.0) 408 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2024-07-15T07:03:06.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:05 smithi089 bash[82202]: audit 2024-07-15T07:03:04.943132+0000 mon.a (mon.0) 409 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.c", "id": [4, 2]}]': finished 2024-07-15T07:03:06.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:05 smithi089 bash[82202]: audit 2024-07-15T07:03:04.943234+0000 mon.a (mon.0) 410 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 0]}]': finished 2024-07-15T07:03:06.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:05 smithi089 bash[82202]: audit 2024-07-15T07:03:04.943372+0000 mon.a (mon.0) 411 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]': finished 2024-07-15T07:03:06.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:05 smithi089 bash[82202]: audit 2024-07-15T07:03:04.943494+0000 mon.a (mon.0) 412 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [3, 2]}]': finished 2024-07-15T07:03:06.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:05 smithi089 bash[82202]: audit 2024-07-15T07:03:04.943663+0000 mon.a (mon.0) 413 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 0, 3, 2]}]': finished 2024-07-15T07:03:06.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:05 smithi089 bash[82202]: audit 2024-07-15T07:03:04.943849+0000 mon.a (mon.0) 414 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [3, 5, 4, 2]}]': finished 2024-07-15T07:03:06.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:05 smithi089 bash[82202]: audit 2024-07-15T07:03:04.944054+0000 mon.a (mon.0) 415 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1e", "id": [3, 2]}]': finished 2024-07-15T07:03:06.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:05 smithi089 bash[82202]: cluster 2024-07-15T07:03:04.944998+0000 mon.a (mon.0) 416 : cluster [DBG] osdmap e92: 8 total, 7 up, 8 in 2024-07-15T07:03:06.123 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:05 smithi089 bash[82202]: cluster 2024-07-15T07:03:05.794442+0000 mon.a (mon.0) 417 : cluster [WRN] Health check failed: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded (PG_DEGRADED) 2024-07-15T07:03:06.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:05 smithi134 bash[67624]: audit 2024-07-15T07:03:04.942979+0000 mon.a (mon.0) 408 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2024-07-15T07:03:06.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:05 smithi134 bash[67624]: audit 2024-07-15T07:03:04.943132+0000 mon.a (mon.0) 409 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.c", "id": [4, 2]}]': finished 2024-07-15T07:03:06.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:05 smithi134 bash[67624]: audit 2024-07-15T07:03:04.943234+0000 mon.a (mon.0) 410 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 0]}]': finished 2024-07-15T07:03:06.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:05 smithi134 bash[67624]: audit 2024-07-15T07:03:04.943372+0000 mon.a (mon.0) 411 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]': finished 2024-07-15T07:03:06.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:05 smithi134 bash[67624]: audit 2024-07-15T07:03:04.943494+0000 mon.a (mon.0) 412 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [3, 2]}]': finished 2024-07-15T07:03:06.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:05 smithi134 bash[67624]: audit 2024-07-15T07:03:04.943663+0000 mon.a (mon.0) 413 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 0, 3, 2]}]': finished 2024-07-15T07:03:06.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:05 smithi134 bash[67624]: audit 2024-07-15T07:03:04.943849+0000 mon.a (mon.0) 414 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [3, 5, 4, 2]}]': finished 2024-07-15T07:03:06.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:05 smithi134 bash[67624]: audit 2024-07-15T07:03:04.944054+0000 mon.a (mon.0) 415 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1e", "id": [3, 2]}]': finished 2024-07-15T07:03:06.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:05 smithi134 bash[67624]: cluster 2024-07-15T07:03:04.944998+0000 mon.a (mon.0) 416 : cluster [DBG] osdmap e92: 8 total, 7 up, 8 in 2024-07-15T07:03:06.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:05 smithi134 bash[67624]: cluster 2024-07-15T07:03:05.794442+0000 mon.a (mon.0) 417 : cluster [WRN] Health check failed: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded (PG_DEGRADED) 2024-07-15T07:03:07.215 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:06 smithi089 bash[82202]: cluster 2024-07-15T07:03:05.950090+0000 mon.a (mon.0) 418 : cluster [DBG] osdmap e93: 8 total, 7 up, 8 in 2024-07-15T07:03:07.216 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:06 smithi089 bash[82202]: cluster 2024-07-15T07:03:06.949800+0000 mon.a (mon.0) 419 : cluster [DBG] osdmap e94: 8 total, 7 up, 8 in 2024-07-15T07:03:07.216 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:06 smithi089 bash[79810]: cluster 2024-07-15T07:03:05.950090+0000 mon.a (mon.0) 418 : cluster [DBG] osdmap e93: 8 total, 7 up, 8 in 2024-07-15T07:03:07.216 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:06 smithi089 bash[79810]: cluster 2024-07-15T07:03:06.949800+0000 mon.a (mon.0) 419 : cluster [DBG] osdmap e94: 8 total, 7 up, 8 in 2024-07-15T07:03:07.216 INFO:journalctl@ceph.osd.2.smithi089.stdout:Jul 15 07:03:07 smithi089 systemd[1]: ceph-75ba02bc-4274-11ef-bcad-c7b262605968@osd.2.service: Deactivated successfully. 2024-07-15T07:03:07.216 INFO:journalctl@ceph.osd.2.smithi089.stdout:Jul 15 07:03:07 smithi089 systemd[1]: Stopped Ceph osd.2 for 75ba02bc-4274-11ef-bcad-c7b262605968. 2024-07-15T07:03:07.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:06 smithi134 bash[67624]: cluster 2024-07-15T07:03:05.950090+0000 mon.a (mon.0) 418 : cluster [DBG] osdmap e93: 8 total, 7 up, 8 in 2024-07-15T07:03:07.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:06 smithi134 bash[67624]: cluster 2024-07-15T07:03:06.949800+0000 mon.a (mon.0) 419 : cluster [DBG] osdmap e94: 8 total, 7 up, 8 in 2024-07-15T07:03:07.474 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:07 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:03:07.475 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 07:03:07 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:03:07.475 INFO:journalctl@ceph.osd.1.smithi089.stdout:Jul 15 07:03:07 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:03:07.475 INFO:journalctl@ceph.osd.3.smithi089.stdout:Jul 15 07:03:07 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:03:07.476 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 07:03:07 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:03:07.476 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:07 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:03:07.476 INFO:journalctl@ceph.osd.0.smithi089.stdout:Jul 15 07:03:07 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:03:07.477 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 07:03:07 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:03:07.477 INFO:journalctl@ceph.osd.2.smithi089.stdout:Jul 15 07:03:07 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:03:07.870 INFO:journalctl@ceph.osd.2.smithi089.stdout:Jul 15 07:03:07 smithi089 systemd[1]: Started Ceph osd.2 for 75ba02bc-4274-11ef-bcad-c7b262605968. 2024-07-15T07:03:08.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:07 smithi134 bash[67624]: cluster 2024-07-15T07:03:06.843812+0000 mgr.y (mgr.25244) 260 : cluster [DBG] pgmap v198: 73 pgs: 1 remapped+peering, 4 peering, 16 active+undersized, 2 active+undersized+degraded, 50 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-07-15T07:03:08.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:07 smithi134 bash[67624]: cluster 2024-07-15T07:03:06.957113+0000 mon.a (mon.0) 420 : cluster [WRN] Health check failed: Reduced data availability: 4 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2024-07-15T07:03:08.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:07 smithi134 bash[67624]: audit 2024-07-15T07:03:07.519548+0000 mon.a (mon.0) 421 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:03:08.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:07 smithi134 bash[67624]: audit 2024-07-15T07:03:07.526139+0000 mon.a (mon.0) 422 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:03:08.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:07 smithi134 bash[67624]: audit 2024-07-15T07:03:07.527274+0000 mon.a (mon.0) 423 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T07:03:08.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:07 smithi134 bash[67624]: audit 2024-07-15T07:03:07.778986+0000 mon.c (mon.2) 6 : audit [DBG] from='client.? 172.21.15.89:0/3586349537' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-15T07:03:08.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:07 smithi089 bash[79810]: cluster 2024-07-15T07:03:06.843812+0000 mgr.y (mgr.25244) 260 : cluster [DBG] pgmap v198: 73 pgs: 1 remapped+peering, 4 peering, 16 active+undersized, 2 active+undersized+degraded, 50 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-07-15T07:03:08.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:07 smithi089 bash[79810]: cluster 2024-07-15T07:03:06.957113+0000 mon.a (mon.0) 420 : cluster [WRN] Health check failed: Reduced data availability: 4 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2024-07-15T07:03:08.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:07 smithi089 bash[79810]: audit 2024-07-15T07:03:07.519548+0000 mon.a (mon.0) 421 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:03:08.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:07 smithi089 bash[79810]: audit 2024-07-15T07:03:07.526139+0000 mon.a (mon.0) 422 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:03:08.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:07 smithi089 bash[79810]: audit 2024-07-15T07:03:07.527274+0000 mon.a (mon.0) 423 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T07:03:08.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:07 smithi089 bash[79810]: audit 2024-07-15T07:03:07.778986+0000 mon.c (mon.2) 6 : audit [DBG] from='client.? 172.21.15.89:0/3586349537' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-15T07:03:08.371 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:07 smithi089 bash[82202]: cluster 2024-07-15T07:03:06.843812+0000 mgr.y (mgr.25244) 260 : cluster [DBG] pgmap v198: 73 pgs: 1 remapped+peering, 4 peering, 16 active+undersized, 2 active+undersized+degraded, 50 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-07-15T07:03:08.371 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:07 smithi089 bash[82202]: cluster 2024-07-15T07:03:06.957113+0000 mon.a (mon.0) 420 : cluster [WRN] Health check failed: Reduced data availability: 4 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2024-07-15T07:03:08.371 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:07 smithi089 bash[82202]: audit 2024-07-15T07:03:07.519548+0000 mon.a (mon.0) 421 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:03:08.371 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:07 smithi089 bash[82202]: audit 2024-07-15T07:03:07.526139+0000 mon.a (mon.0) 422 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:03:08.372 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:07 smithi089 bash[82202]: audit 2024-07-15T07:03:07.527274+0000 mon.a (mon.0) 423 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T07:03:08.372 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:07 smithi089 bash[82202]: audit 2024-07-15T07:03:07.778986+0000 mon.c (mon.2) 6 : audit [DBG] from='client.? 172.21.15.89:0/3586349537' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-15T07:03:09.616 INFO:journalctl@ceph.osd.2.smithi089.stdout:Jul 15 07:03:09 smithi089 bash[98137]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-07-15T07:03:09.617 INFO:journalctl@ceph.osd.2.smithi089.stdout:Jul 15 07:03:09 smithi089 bash[98137]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-07-15T07:03:09.617 INFO:journalctl@ceph.osd.2.smithi089.stdout:Jul 15 07:03:09 smithi089 bash[98137]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-07-15T07:03:09.617 INFO:journalctl@ceph.osd.2.smithi089.stdout:Jul 15 07:03:09 smithi089 bash[98137]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-07-15T07:03:09.617 INFO:journalctl@ceph.osd.2.smithi089.stdout:Jul 15 07:03:09 smithi089 bash[98137]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-07-15T07:03:09.617 INFO:journalctl@ceph.osd.2.smithi089.stdout:Jul 15 07:03:09 smithi089 bash[98137]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-07-15T07:03:09.617 INFO:journalctl@ceph.osd.2.smithi089.stdout:Jul 15 07:03:09 smithi089 bash[98137]: --> ceph-volume raw activate successful for osd ID: 2 2024-07-15T07:03:10.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:09 smithi134 bash[67624]: cluster 2024-07-15T07:03:08.844786+0000 mgr.y (mgr.25244) 261 : cluster [DBG] pgmap v200: 73 pgs: 1 active+clean+remapped, 1 remapped+peering, 4 peering, 16 active+undersized, 2 active+undersized+degraded, 49 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-07-15T07:03:10.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:09 smithi089 bash[79810]: cluster 2024-07-15T07:03:08.844786+0000 mgr.y (mgr.25244) 261 : cluster [DBG] pgmap v200: 73 pgs: 1 active+clean+remapped, 1 remapped+peering, 4 peering, 16 active+undersized, 2 active+undersized+degraded, 49 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-07-15T07:03:10.371 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:09 smithi089 bash[82202]: cluster 2024-07-15T07:03:08.844786+0000 mgr.y (mgr.25244) 261 : cluster [DBG] pgmap v200: 73 pgs: 1 active+clean+remapped, 1 remapped+peering, 4 peering, 16 active+undersized, 2 active+undersized+degraded, 49 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-07-15T07:03:11.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:10 smithi134 bash[67624]: audit 2024-07-15T07:03:10.090034+0000 mon.a (mon.0) 424 : audit [DBG] from='client.? 172.21.15.89:0/2753278719' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-15T07:03:11.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:10 smithi089 bash[79810]: audit 2024-07-15T07:03:10.090034+0000 mon.a (mon.0) 424 : audit [DBG] from='client.? 172.21.15.89:0/2753278719' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-15T07:03:11.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:10 smithi089 bash[82202]: audit 2024-07-15T07:03:10.090034+0000 mon.a (mon.0) 424 : audit [DBG] from='client.? 172.21.15.89:0/2753278719' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-15T07:03:11.371 INFO:journalctl@ceph.osd.2.smithi089.stdout:Jul 15 07:03:11 smithi089 bash[98398]: debug 2024-07-15T07:03:11.190+0000 7f0f7e0d0640 1 -- 172.21.15.89:0/190866498 <== mon.0 v2:172.21.15.89:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5606cb3eb1e0 con 0x5606cc168000 2024-07-15T07:03:12.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:11 smithi089 bash[82202]: cluster 2024-07-15T07:03:10.845891+0000 mgr.y (mgr.25244) 262 : cluster [DBG] pgmap v201: 73 pgs: 6 active+clean+remapped, 1 remapped+peering, 16 active+undersized, 2 active+undersized+degraded, 48 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-07-15T07:03:12.120 INFO:journalctl@ceph.osd.2.smithi089.stdout:Jul 15 07:03:11 smithi089 bash[98398]: debug 2024-07-15T07:03:11.726+0000 7f0f80936640 -1 Falling back to public interface 2024-07-15T07:03:12.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:11 smithi089 bash[79810]: cluster 2024-07-15T07:03:10.845891+0000 mgr.y (mgr.25244) 262 : cluster [DBG] pgmap v201: 73 pgs: 6 active+clean+remapped, 1 remapped+peering, 16 active+undersized, 2 active+undersized+degraded, 48 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-07-15T07:03:12.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:11 smithi134 bash[67624]: cluster 2024-07-15T07:03:10.845891+0000 mgr.y (mgr.25244) 262 : cluster [DBG] pgmap v201: 73 pgs: 6 active+clean+remapped, 1 remapped+peering, 16 active+undersized, 2 active+undersized+degraded, 48 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-07-15T07:03:12.804 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:03:12 smithi134 bash[60729]: ts=2024-07-15T07:03:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T07:03:13.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:12 smithi134 bash[67624]: cluster 2024-07-15T07:03:12.295905+0000 mon.a (mon.0) 425 : cluster [WRN] Health check update: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-07-15T07:03:13.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:12 smithi134 bash[67624]: cluster 2024-07-15T07:03:12.295952+0000 mon.a (mon.0) 426 : cluster [WRN] Health check update: Degraded data redundancy: 4/72 objects degraded (5.556%), 2 pgs degraded (PG_DEGRADED) 2024-07-15T07:03:13.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:12 smithi134 bash[67624]: audit 2024-07-15T07:03:12.393341+0000 mon.a (mon.0) 427 : audit [DBG] from='client.? 172.21.15.89:0/3630663721' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-15T07:03:13.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:12 smithi089 bash[79810]: cluster 2024-07-15T07:03:12.295905+0000 mon.a (mon.0) 425 : cluster [WRN] Health check update: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-07-15T07:03:13.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:12 smithi089 bash[79810]: cluster 2024-07-15T07:03:12.295952+0000 mon.a (mon.0) 426 : cluster [WRN] Health check update: Degraded data redundancy: 4/72 objects degraded (5.556%), 2 pgs degraded (PG_DEGRADED) 2024-07-15T07:03:13.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:12 smithi089 bash[79810]: audit 2024-07-15T07:03:12.393341+0000 mon.a (mon.0) 427 : audit [DBG] from='client.? 172.21.15.89:0/3630663721' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-15T07:03:13.371 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:12 smithi089 bash[82202]: cluster 2024-07-15T07:03:12.295905+0000 mon.a (mon.0) 425 : cluster [WRN] Health check update: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-07-15T07:03:13.371 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:12 smithi089 bash[82202]: cluster 2024-07-15T07:03:12.295952+0000 mon.a (mon.0) 426 : cluster [WRN] Health check update: Degraded data redundancy: 4/72 objects degraded (5.556%), 2 pgs degraded (PG_DEGRADED) 2024-07-15T07:03:13.371 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:12 smithi089 bash[82202]: audit 2024-07-15T07:03:12.393341+0000 mon.a (mon.0) 427 : audit [DBG] from='client.? 172.21.15.89:0/3630663721' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-15T07:03:14.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:13 smithi134 bash[67624]: cluster 2024-07-15T07:03:12.846783+0000 mgr.y (mgr.25244) 263 : cluster [DBG] pgmap v202: 73 pgs: 7 active+clean+remapped, 16 active+undersized, 2 active+undersized+degraded, 48 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-07-15T07:03:14.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:13 smithi134 bash[67624]: cluster 2024-07-15T07:03:12.976871+0000 mon.a (mon.0) 428 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-07-15T07:03:14.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:13 smithi134 bash[67624]: audit 2024-07-15T07:03:13.275990+0000 mon.a (mon.0) 429 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:03:14.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:13 smithi134 bash[67624]: audit 2024-07-15T07:03:13.288126+0000 mon.a (mon.0) 430 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:03:14.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:13 smithi089 bash[82202]: cluster 2024-07-15T07:03:12.846783+0000 mgr.y (mgr.25244) 263 : cluster [DBG] pgmap v202: 73 pgs: 7 active+clean+remapped, 16 active+undersized, 2 active+undersized+degraded, 48 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-07-15T07:03:14.371 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:13 smithi089 bash[82202]: cluster 2024-07-15T07:03:12.976871+0000 mon.a (mon.0) 428 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-07-15T07:03:14.371 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:13 smithi089 bash[82202]: audit 2024-07-15T07:03:13.275990+0000 mon.a (mon.0) 429 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:03:14.371 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:13 smithi089 bash[82202]: audit 2024-07-15T07:03:13.288126+0000 mon.a (mon.0) 430 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:03:14.371 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 07:03:14 smithi089 bash[72894]: ::ffff:172.21.15.134 - - [15/Jul/2024:07:03:14] "GET /metrics HTTP/1.1" 200 33984 "" "Prometheus/2.43.0" 2024-07-15T07:03:14.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:13 smithi089 bash[79810]: cluster 2024-07-15T07:03:12.846783+0000 mgr.y (mgr.25244) 263 : cluster [DBG] pgmap v202: 73 pgs: 7 active+clean+remapped, 16 active+undersized, 2 active+undersized+degraded, 48 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-07-15T07:03:14.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:13 smithi089 bash[79810]: cluster 2024-07-15T07:03:12.976871+0000 mon.a (mon.0) 428 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-07-15T07:03:14.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:13 smithi089 bash[79810]: audit 2024-07-15T07:03:13.275990+0000 mon.a (mon.0) 429 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:03:14.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:13 smithi089 bash[79810]: audit 2024-07-15T07:03:13.288126+0000 mon.a (mon.0) 430 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:03:15.120 INFO:journalctl@ceph.osd.2.smithi089.stdout:Jul 15 07:03:14 smithi089 bash[98398]: debug 2024-07-15T07:03:14.734+0000 7f0f80936640 -1 osd.2 89 log_to_monitors true 2024-07-15T07:03:15.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:15 smithi134 bash[67624]: audit 2024-07-15T07:03:14.234034+0000 mon.a (mon.0) 431 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:03:15.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:15 smithi134 bash[67624]: audit 2024-07-15T07:03:14.239964+0000 mon.a (mon.0) 432 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:03:15.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:15 smithi134 bash[67624]: audit 2024-07-15T07:03:14.669451+0000 mon.a (mon.0) 433 : audit [DBG] from='client.? 172.21.15.89:0/1884750829' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-15T07:03:15.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:15 smithi134 bash[67624]: audit 2024-07-15T07:03:14.738689+0000 mon.a (mon.0) 434 : audit [INF] from='osd.2 [v2:172.21.15.89:6818/443062433,v1:172.21.15.89:6819/443062433]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-07-15T07:03:15.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:15 smithi134 bash[67624]: audit 2024-07-15T07:03:14.896383+0000 mon.a (mon.0) 435 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:03:15.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:15 smithi134 bash[67624]: audit 2024-07-15T07:03:14.901660+0000 mon.a (mon.0) 436 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:03:15.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:15 smithi089 bash[82202]: audit 2024-07-15T07:03:14.234034+0000 mon.a (mon.0) 431 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:03:15.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:15 smithi089 bash[82202]: audit 2024-07-15T07:03:14.239964+0000 mon.a (mon.0) 432 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:03:15.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:15 smithi089 bash[82202]: audit 2024-07-15T07:03:14.669451+0000 mon.a (mon.0) 433 : audit [DBG] from='client.? 172.21.15.89:0/1884750829' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-15T07:03:15.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:15 smithi089 bash[82202]: audit 2024-07-15T07:03:14.738689+0000 mon.a (mon.0) 434 : audit [INF] from='osd.2 [v2:172.21.15.89:6818/443062433,v1:172.21.15.89:6819/443062433]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-07-15T07:03:15.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:15 smithi089 bash[82202]: audit 2024-07-15T07:03:14.896383+0000 mon.a (mon.0) 435 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:03:15.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:15 smithi089 bash[82202]: audit 2024-07-15T07:03:14.901660+0000 mon.a (mon.0) 436 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:03:15.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:15 smithi089 bash[79810]: audit 2024-07-15T07:03:14.234034+0000 mon.a (mon.0) 431 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:03:15.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:15 smithi089 bash[79810]: audit 2024-07-15T07:03:14.239964+0000 mon.a (mon.0) 432 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:03:15.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:15 smithi089 bash[79810]: audit 2024-07-15T07:03:14.669451+0000 mon.a (mon.0) 433 : audit [DBG] from='client.? 172.21.15.89:0/1884750829' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-15T07:03:15.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:15 smithi089 bash[79810]: audit 2024-07-15T07:03:14.738689+0000 mon.a (mon.0) 434 : audit [INF] from='osd.2 [v2:172.21.15.89:6818/443062433,v1:172.21.15.89:6819/443062433]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-07-15T07:03:15.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:15 smithi089 bash[79810]: audit 2024-07-15T07:03:14.896383+0000 mon.a (mon.0) 435 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:03:15.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:15 smithi089 bash[79810]: audit 2024-07-15T07:03:14.901660+0000 mon.a (mon.0) 436 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:03:16.121 INFO:journalctl@ceph.osd.2.smithi089.stdout:Jul 15 07:03:15 smithi089 bash[98398]: debug 2024-07-15T07:03:15.790+0000 7f0f746d8640 -1 osd.2 89 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-07-15T07:03:16.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:16 smithi134 bash[67624]: cluster 2024-07-15T07:03:14.847613+0000 mgr.y (mgr.25244) 264 : cluster [DBG] pgmap v203: 73 pgs: 7 active+clean+remapped, 16 active+undersized, 2 active+undersized+degraded, 48 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-07-15T07:03:16.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:16 smithi134 bash[67624]: audit 2024-07-15T07:03:15.258010+0000 mon.a (mon.0) 437 : audit [INF] from='osd.2 [v2:172.21.15.89:6818/443062433,v1:172.21.15.89:6819/443062433]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-07-15T07:03:16.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:16 smithi134 bash[67624]: cluster 2024-07-15T07:03:15.258102+0000 mon.a (mon.0) 438 : cluster [DBG] osdmap e95: 8 total, 7 up, 8 in 2024-07-15T07:03:16.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:16 smithi134 bash[67624]: audit 2024-07-15T07:03:15.259115+0000 mon.a (mon.0) 439 : audit [INF] from='osd.2 [v2:172.21.15.89:6818/443062433,v1:172.21.15.89:6819/443062433]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2024-07-15T07:03:16.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:16 smithi134 bash[67624]: audit 2024-07-15T07:03:15.780140+0000 mon.a (mon.0) 440 : audit [INF] from='osd.2 [v2:172.21.15.89:6818/443062433,v1:172.21.15.89:6819/443062433]' entity='osd.2' 2024-07-15T07:03:16.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:16 smithi089 bash[79810]: cluster 2024-07-15T07:03:14.847613+0000 mgr.y (mgr.25244) 264 : cluster [DBG] pgmap v203: 73 pgs: 7 active+clean+remapped, 16 active+undersized, 2 active+undersized+degraded, 48 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-07-15T07:03:16.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:16 smithi089 bash[79810]: audit 2024-07-15T07:03:15.258010+0000 mon.a (mon.0) 437 : audit [INF] from='osd.2 [v2:172.21.15.89:6818/443062433,v1:172.21.15.89:6819/443062433]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-07-15T07:03:16.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:16 smithi089 bash[79810]: cluster 2024-07-15T07:03:15.258102+0000 mon.a (mon.0) 438 : cluster [DBG] osdmap e95: 8 total, 7 up, 8 in 2024-07-15T07:03:16.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:16 smithi089 bash[79810]: audit 2024-07-15T07:03:15.259115+0000 mon.a (mon.0) 439 : audit [INF] from='osd.2 [v2:172.21.15.89:6818/443062433,v1:172.21.15.89:6819/443062433]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2024-07-15T07:03:16.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:16 smithi089 bash[79810]: audit 2024-07-15T07:03:15.780140+0000 mon.a (mon.0) 440 : audit [INF] from='osd.2 [v2:172.21.15.89:6818/443062433,v1:172.21.15.89:6819/443062433]' entity='osd.2' 2024-07-15T07:03:16.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:16 smithi089 bash[82202]: cluster 2024-07-15T07:03:14.847613+0000 mgr.y (mgr.25244) 264 : cluster [DBG] pgmap v203: 73 pgs: 7 active+clean+remapped, 16 active+undersized, 2 active+undersized+degraded, 48 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-07-15T07:03:16.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:16 smithi089 bash[82202]: audit 2024-07-15T07:03:15.258010+0000 mon.a (mon.0) 437 : audit [INF] from='osd.2 [v2:172.21.15.89:6818/443062433,v1:172.21.15.89:6819/443062433]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-07-15T07:03:16.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:16 smithi089 bash[82202]: cluster 2024-07-15T07:03:15.258102+0000 mon.a (mon.0) 438 : cluster [DBG] osdmap e95: 8 total, 7 up, 8 in 2024-07-15T07:03:16.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:16 smithi089 bash[82202]: audit 2024-07-15T07:03:15.259115+0000 mon.a (mon.0) 439 : audit [INF] from='osd.2 [v2:172.21.15.89:6818/443062433,v1:172.21.15.89:6819/443062433]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2024-07-15T07:03:16.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:16 smithi089 bash[82202]: audit 2024-07-15T07:03:15.780140+0000 mon.a (mon.0) 440 : audit [INF] from='osd.2 [v2:172.21.15.89:6818/443062433,v1:172.21.15.89:6819/443062433]' entity='osd.2' 2024-07-15T07:03:17.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:16 smithi089 bash[79810]: debug 2024-07-15T07:03:16.774+0000 7fb563a0a640 -1 log_channel(cluster) log [ERR] : Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-07-15T07:03:17.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:17 smithi134 bash[67624]: cluster 2024-07-15T07:03:16.775337+0000 mon.a (mon.0) 441 : cluster [ERR] Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-07-15T07:03:17.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:17 smithi134 bash[67624]: cluster 2024-07-15T07:03:16.775373+0000 mon.a (mon.0) 442 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-07-15T07:03:17.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:17 smithi134 bash[67624]: cluster 2024-07-15T07:03:16.780821+0000 mon.a (mon.0) 443 : cluster [INF] osd.2 [v2:172.21.15.89:6818/443062433,v1:172.21.15.89:6819/443062433] boot 2024-07-15T07:03:17.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:17 smithi134 bash[67624]: cluster 2024-07-15T07:03:16.780887+0000 mon.a (mon.0) 444 : cluster [DBG] osdmap e96: 8 total, 8 up, 8 in 2024-07-15T07:03:17.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:17 smithi134 bash[67624]: audit 2024-07-15T07:03:16.781520+0000 mon.a (mon.0) 445 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-07-15T07:03:17.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:17 smithi134 bash[67624]: audit 2024-07-15T07:03:16.986426+0000 mon.c (mon.2) 7 : audit [DBG] from='client.? 172.21.15.89:0/2879170504' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-15T07:03:17.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:17 smithi089 bash[79810]: cluster 2024-07-15T07:03:16.775337+0000 mon.a (mon.0) 441 : cluster [ERR] Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-07-15T07:03:17.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:17 smithi089 bash[79810]: cluster 2024-07-15T07:03:16.775373+0000 mon.a (mon.0) 442 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-07-15T07:03:17.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:17 smithi089 bash[79810]: cluster 2024-07-15T07:03:16.780821+0000 mon.a (mon.0) 443 : cluster [INF] osd.2 [v2:172.21.15.89:6818/443062433,v1:172.21.15.89:6819/443062433] boot 2024-07-15T07:03:17.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:17 smithi089 bash[79810]: cluster 2024-07-15T07:03:16.780887+0000 mon.a (mon.0) 444 : cluster [DBG] osdmap e96: 8 total, 8 up, 8 in 2024-07-15T07:03:17.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:17 smithi089 bash[79810]: audit 2024-07-15T07:03:16.781520+0000 mon.a (mon.0) 445 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-07-15T07:03:17.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:17 smithi089 bash[79810]: audit 2024-07-15T07:03:16.986426+0000 mon.c (mon.2) 7 : audit [DBG] from='client.? 172.21.15.89:0/2879170504' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-15T07:03:17.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:17 smithi089 bash[82202]: cluster 2024-07-15T07:03:16.775337+0000 mon.a (mon.0) 441 : cluster [ERR] Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-07-15T07:03:17.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:17 smithi089 bash[82202]: cluster 2024-07-15T07:03:16.775373+0000 mon.a (mon.0) 442 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-07-15T07:03:17.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:17 smithi089 bash[82202]: cluster 2024-07-15T07:03:16.780821+0000 mon.a (mon.0) 443 : cluster [INF] osd.2 [v2:172.21.15.89:6818/443062433,v1:172.21.15.89:6819/443062433] boot 2024-07-15T07:03:17.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:17 smithi089 bash[82202]: cluster 2024-07-15T07:03:16.780887+0000 mon.a (mon.0) 444 : cluster [DBG] osdmap e96: 8 total, 8 up, 8 in 2024-07-15T07:03:17.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:17 smithi089 bash[82202]: audit 2024-07-15T07:03:16.781520+0000 mon.a (mon.0) 445 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-07-15T07:03:17.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:17 smithi089 bash[82202]: audit 2024-07-15T07:03:16.986426+0000 mon.c (mon.2) 7 : audit [DBG] from='client.? 172.21.15.89:0/2879170504' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-15T07:03:18.554 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:18 smithi134 bash[67624]: cluster 2024-07-15T07:03:16.848243+0000 mgr.y (mgr.25244) 265 : cluster [DBG] pgmap v206: 73 pgs: 7 active+clean+remapped, 16 active+undersized, 2 active+undersized+degraded, 48 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-07-15T07:03:18.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:18 smithi134 bash[67624]: cluster 2024-07-15T07:03:17.307310+0000 mon.a (mon.0) 446 : cluster [DBG] osdmap e97: 8 total, 8 up, 8 in 2024-07-15T07:03:18.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:18 smithi089 bash[82202]: cluster 2024-07-15T07:03:16.848243+0000 mgr.y (mgr.25244) 265 : cluster [DBG] pgmap v206: 73 pgs: 7 active+clean+remapped, 16 active+undersized, 2 active+undersized+degraded, 48 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-07-15T07:03:18.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:18 smithi089 bash[82202]: cluster 2024-07-15T07:03:17.307310+0000 mon.a (mon.0) 446 : cluster [DBG] osdmap e97: 8 total, 8 up, 8 in 2024-07-15T07:03:18.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:18 smithi089 bash[79810]: cluster 2024-07-15T07:03:16.848243+0000 mgr.y (mgr.25244) 265 : cluster [DBG] pgmap v206: 73 pgs: 7 active+clean+remapped, 16 active+undersized, 2 active+undersized+degraded, 48 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-07-15T07:03:18.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:18 smithi089 bash[79810]: cluster 2024-07-15T07:03:17.307310+0000 mon.a (mon.0) 446 : cluster [DBG] osdmap e97: 8 total, 8 up, 8 in 2024-07-15T07:03:19.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:19 smithi089 bash[79810]: cluster 2024-07-15T07:03:18.316291+0000 mon.a (mon.0) 447 : cluster [DBG] osdmap e98: 8 total, 8 up, 8 in 2024-07-15T07:03:19.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:19 smithi089 bash[79810]: audit 2024-07-15T07:03:19.290972+0000 mon.a (mon.0) 448 : audit [DBG] from='client.? 172.21.15.89:0/2099055926' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-15T07:03:19.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:19 smithi089 bash[82202]: cluster 2024-07-15T07:03:18.316291+0000 mon.a (mon.0) 447 : cluster [DBG] osdmap e98: 8 total, 8 up, 8 in 2024-07-15T07:03:19.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:19 smithi089 bash[82202]: audit 2024-07-15T07:03:19.290972+0000 mon.a (mon.0) 448 : audit [DBG] from='client.? 172.21.15.89:0/2099055926' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-15T07:03:19.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:19 smithi134 bash[67624]: cluster 2024-07-15T07:03:18.316291+0000 mon.a (mon.0) 447 : cluster [DBG] osdmap e98: 8 total, 8 up, 8 in 2024-07-15T07:03:19.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:19 smithi134 bash[67624]: audit 2024-07-15T07:03:19.290972+0000 mon.a (mon.0) 448 : audit [DBG] from='client.? 172.21.15.89:0/2099055926' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-15T07:03:20.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:20 smithi089 bash[82202]: cluster 2024-07-15T07:03:18.849128+0000 mgr.y (mgr.25244) 266 : cluster [DBG] pgmap v209: 73 pgs: 1 activating, 6 active+clean+remapped, 15 active+undersized, 1 active+undersized+degraded, 50 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2024-07-15T07:03:20.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:20 smithi089 bash[82202]: cluster 2024-07-15T07:03:19.309901+0000 mon.a (mon.0) 449 : cluster [WRN] Health check update: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded (PG_DEGRADED) 2024-07-15T07:03:20.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:20 smithi089 bash[82202]: audit 2024-07-15T07:03:19.797681+0000 mon.a (mon.0) 450 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:03:20.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:20 smithi089 bash[82202]: audit 2024-07-15T07:03:19.798810+0000 mon.a (mon.0) 451 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:03:20.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:20 smithi089 bash[79810]: cluster 2024-07-15T07:03:18.849128+0000 mgr.y (mgr.25244) 266 : cluster [DBG] pgmap v209: 73 pgs: 1 activating, 6 active+clean+remapped, 15 active+undersized, 1 active+undersized+degraded, 50 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2024-07-15T07:03:20.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:20 smithi089 bash[79810]: cluster 2024-07-15T07:03:19.309901+0000 mon.a (mon.0) 449 : cluster [WRN] Health check update: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded (PG_DEGRADED) 2024-07-15T07:03:20.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:20 smithi089 bash[79810]: audit 2024-07-15T07:03:19.797681+0000 mon.a (mon.0) 450 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:03:20.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:20 smithi089 bash[79810]: audit 2024-07-15T07:03:19.798810+0000 mon.a (mon.0) 451 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:03:20.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:20 smithi134 bash[67624]: cluster 2024-07-15T07:03:18.849128+0000 mgr.y (mgr.25244) 266 : cluster [DBG] pgmap v209: 73 pgs: 1 activating, 6 active+clean+remapped, 15 active+undersized, 1 active+undersized+degraded, 50 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2024-07-15T07:03:20.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:20 smithi134 bash[67624]: cluster 2024-07-15T07:03:19.309901+0000 mon.a (mon.0) 449 : cluster [WRN] Health check update: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded (PG_DEGRADED) 2024-07-15T07:03:20.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:20 smithi134 bash[67624]: audit 2024-07-15T07:03:19.797681+0000 mon.a (mon.0) 450 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:03:20.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:20 smithi134 bash[67624]: audit 2024-07-15T07:03:19.798810+0000 mon.a (mon.0) 451 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:03:21.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:21 smithi089 bash[79810]: cluster 2024-07-15T07:03:21.319843+0000 mon.a (mon.0) 452 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded) 2024-07-15T07:03:21.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:21 smithi089 bash[82202]: cluster 2024-07-15T07:03:21.319843+0000 mon.a (mon.0) 452 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded) 2024-07-15T07:03:21.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:21 smithi134 bash[67624]: cluster 2024-07-15T07:03:21.319843+0000 mon.a (mon.0) 452 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded) 2024-07-15T07:03:22.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:22 smithi089 bash[79810]: cluster 2024-07-15T07:03:20.850117+0000 mgr.y (mgr.25244) 267 : cluster [DBG] pgmap v210: 73 pgs: 1 activating, 3 active+undersized, 69 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:03:22.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:22 smithi089 bash[79810]: audit 2024-07-15T07:03:21.575265+0000 mon.c (mon.2) 8 : audit [DBG] from='client.? 172.21.15.89:0/336705122' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-15T07:03:22.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:22 smithi089 bash[82202]: cluster 2024-07-15T07:03:20.850117+0000 mgr.y (mgr.25244) 267 : cluster [DBG] pgmap v210: 73 pgs: 1 activating, 3 active+undersized, 69 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:03:22.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:22 smithi089 bash[82202]: audit 2024-07-15T07:03:21.575265+0000 mon.c (mon.2) 8 : audit [DBG] from='client.? 172.21.15.89:0/336705122' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-15T07:03:22.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:22 smithi134 bash[67624]: cluster 2024-07-15T07:03:20.850117+0000 mgr.y (mgr.25244) 267 : cluster [DBG] pgmap v210: 73 pgs: 1 activating, 3 active+undersized, 69 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:03:22.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:22 smithi134 bash[67624]: audit 2024-07-15T07:03:21.575265+0000 mon.c (mon.2) 8 : audit [DBG] from='client.? 172.21.15.89:0/336705122' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-15T07:03:22.805 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:03:22 smithi134 bash[60729]: ts=2024-07-15T07:03:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T07:03:24.398 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 07:03:24 smithi089 bash[72894]: ::ffff:172.21.15.134 - - [15/Jul/2024:07:03:24] "GET /metrics HTTP/1.1" 200 33833 "" "Prometheus/2.43.0" 2024-07-15T07:03:24.398 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:24 smithi089 bash[79810]: cluster 2024-07-15T07:03:22.850818+0000 mgr.y (mgr.25244) 268 : cluster [DBG] pgmap v211: 73 pgs: 1 activating, 72 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:03:24.398 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:24 smithi089 bash[79810]: audit 2024-07-15T07:03:23.879018+0000 mon.a (mon.0) 453 : audit [DBG] from='client.? 172.21.15.89:0/1150980061' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-15T07:03:24.398 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:24 smithi089 bash[82202]: cluster 2024-07-15T07:03:22.850818+0000 mgr.y (mgr.25244) 268 : cluster [DBG] pgmap v211: 73 pgs: 1 activating, 72 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:03:24.398 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:24 smithi089 bash[82202]: audit 2024-07-15T07:03:23.879018+0000 mon.a (mon.0) 453 : audit [DBG] from='client.? 172.21.15.89:0/1150980061' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-15T07:03:24.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:24 smithi134 bash[67624]: cluster 2024-07-15T07:03:22.850818+0000 mgr.y (mgr.25244) 268 : cluster [DBG] pgmap v211: 73 pgs: 1 activating, 72 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:03:24.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:24 smithi134 bash[67624]: audit 2024-07-15T07:03:23.879018+0000 mon.a (mon.0) 453 : audit [DBG] from='client.? 172.21.15.89:0/1150980061' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-15T07:03:26.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:25 smithi134 bash[67624]: cluster 2024-07-15T07:03:24.851647+0000 mgr.y (mgr.25244) 269 : cluster [DBG] pgmap v212: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:03:26.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:25 smithi134 bash[67624]: audit 2024-07-15T07:03:24.953818+0000 mon.a (mon.0) 454 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:03:26.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:25 smithi134 bash[67624]: audit 2024-07-15T07:03:24.961664+0000 mon.a (mon.0) 455 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:03:26.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:25 smithi134 bash[67624]: audit 2024-07-15T07:03:24.963769+0000 mon.a (mon.0) 456 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T07:03:26.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:25 smithi134 bash[67624]: audit 2024-07-15T07:03:24.965566+0000 mon.a (mon.0) 457 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T07:03:26.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:25 smithi134 bash[67624]: audit 2024-07-15T07:03:24.973468+0000 mon.a (mon.0) 458 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:03:26.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:25 smithi134 bash[67624]: audit 2024-07-15T07:03:25.128244+0000 mon.a (mon.0) 459 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T07:03:26.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:25 smithi134 bash[67624]: audit 2024-07-15T07:03:25.133180+0000 mon.a (mon.0) 460 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-15T07:03:26.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:25 smithi134 bash[67624]: audit 2024-07-15T07:03:25.135904+0000 mon.a (mon.0) 461 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-15T07:03:26.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:25 smithi134 bash[67624]: audit 2024-07-15T07:03:25.138119+0000 mon.a (mon.0) 462 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-15T07:03:26.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:25 smithi134 bash[67624]: audit 2024-07-15T07:03:25.140557+0000 mon.a (mon.0) 463 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-07-15T07:03:26.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:25 smithi134 bash[67624]: audit 2024-07-15T07:03:25.140914+0000 mgr.y (mgr.25244) 270 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-07-15T07:03:26.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:25 smithi134 bash[67624]: cephadm 2024-07-15T07:03:25.142541+0000 mgr.y (mgr.25244) 271 : cephadm [INF] Upgrade: osd.3 is safe to restart 2024-07-15T07:03:26.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:25 smithi089 bash[79810]: cluster 2024-07-15T07:03:24.851647+0000 mgr.y (mgr.25244) 269 : cluster [DBG] pgmap v212: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:03:26.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:25 smithi089 bash[79810]: audit 2024-07-15T07:03:24.953818+0000 mon.a (mon.0) 454 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:03:26.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:25 smithi089 bash[79810]: audit 2024-07-15T07:03:24.961664+0000 mon.a (mon.0) 455 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:03:26.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:25 smithi089 bash[79810]: audit 2024-07-15T07:03:24.963769+0000 mon.a (mon.0) 456 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T07:03:26.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:25 smithi089 bash[79810]: audit 2024-07-15T07:03:24.965566+0000 mon.a (mon.0) 457 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T07:03:26.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:25 smithi089 bash[79810]: audit 2024-07-15T07:03:24.973468+0000 mon.a (mon.0) 458 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:03:26.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:25 smithi089 bash[79810]: audit 2024-07-15T07:03:25.128244+0000 mon.a (mon.0) 459 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T07:03:26.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:25 smithi089 bash[79810]: audit 2024-07-15T07:03:25.133180+0000 mon.a (mon.0) 460 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-15T07:03:26.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:25 smithi089 bash[79810]: audit 2024-07-15T07:03:25.135904+0000 mon.a (mon.0) 461 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-15T07:03:26.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:25 smithi089 bash[79810]: audit 2024-07-15T07:03:25.138119+0000 mon.a (mon.0) 462 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-15T07:03:26.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:25 smithi089 bash[79810]: audit 2024-07-15T07:03:25.140557+0000 mon.a (mon.0) 463 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-07-15T07:03:26.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:25 smithi089 bash[79810]: audit 2024-07-15T07:03:25.140914+0000 mgr.y (mgr.25244) 270 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-07-15T07:03:26.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:25 smithi089 bash[79810]: cephadm 2024-07-15T07:03:25.142541+0000 mgr.y (mgr.25244) 271 : cephadm [INF] Upgrade: osd.3 is safe to restart 2024-07-15T07:03:26.372 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:25 smithi089 bash[82202]: cluster 2024-07-15T07:03:24.851647+0000 mgr.y (mgr.25244) 269 : cluster [DBG] pgmap v212: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:03:26.372 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:25 smithi089 bash[82202]: audit 2024-07-15T07:03:24.953818+0000 mon.a (mon.0) 454 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:03:26.372 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:25 smithi089 bash[82202]: audit 2024-07-15T07:03:24.961664+0000 mon.a (mon.0) 455 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:03:26.372 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:25 smithi089 bash[82202]: audit 2024-07-15T07:03:24.963769+0000 mon.a (mon.0) 456 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T07:03:26.372 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:25 smithi089 bash[82202]: audit 2024-07-15T07:03:24.965566+0000 mon.a (mon.0) 457 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T07:03:26.372 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:25 smithi089 bash[82202]: audit 2024-07-15T07:03:24.973468+0000 mon.a (mon.0) 458 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:03:26.372 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:25 smithi089 bash[82202]: audit 2024-07-15T07:03:25.128244+0000 mon.a (mon.0) 459 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T07:03:26.372 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:25 smithi089 bash[82202]: audit 2024-07-15T07:03:25.133180+0000 mon.a (mon.0) 460 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-15T07:03:26.372 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:25 smithi089 bash[82202]: audit 2024-07-15T07:03:25.135904+0000 mon.a (mon.0) 461 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-15T07:03:26.372 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:25 smithi089 bash[82202]: audit 2024-07-15T07:03:25.138119+0000 mon.a (mon.0) 462 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-15T07:03:26.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:25 smithi089 bash[82202]: audit 2024-07-15T07:03:25.140557+0000 mon.a (mon.0) 463 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-07-15T07:03:26.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:25 smithi089 bash[82202]: audit 2024-07-15T07:03:25.140914+0000 mgr.y (mgr.25244) 270 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-07-15T07:03:26.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:25 smithi089 bash[82202]: cephadm 2024-07-15T07:03:25.142541+0000 mgr.y (mgr.25244) 271 : cephadm [INF] Upgrade: osd.3 is safe to restart 2024-07-15T07:03:27.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:26 smithi134 bash[67624]: audit 2024-07-15T07:03:26.187192+0000 mon.a (mon.0) 464 : audit [DBG] from='client.? 172.21.15.89:0/1078005743' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-15T07:03:27.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:26 smithi089 bash[79810]: audit 2024-07-15T07:03:26.187192+0000 mon.a (mon.0) 464 : audit [DBG] from='client.? 172.21.15.89:0/1078005743' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-15T07:03:27.371 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:26 smithi089 bash[82202]: audit 2024-07-15T07:03:26.187192+0000 mon.a (mon.0) 464 : audit [DBG] from='client.? 172.21.15.89:0/1078005743' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-15T07:03:28.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:28 smithi134 bash[67624]: cluster 2024-07-15T07:03:26.852713+0000 mgr.y (mgr.25244) 272 : cluster [DBG] pgmap v213: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:03:28.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:28 smithi134 bash[67624]: cephadm 2024-07-15T07:03:27.442824+0000 mgr.y (mgr.25244) 273 : cephadm [INF] Upgrade: Updating osd.3 2024-07-15T07:03:28.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:28 smithi134 bash[67624]: audit 2024-07-15T07:03:27.451469+0000 mon.a (mon.0) 465 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:03:28.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:28 smithi134 bash[67624]: audit 2024-07-15T07:03:27.452928+0000 mon.a (mon.0) 466 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-07-15T07:03:28.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:28 smithi134 bash[67624]: audit 2024-07-15T07:03:27.453899+0000 mon.a (mon.0) 467 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T07:03:28.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:28 smithi134 bash[67624]: cephadm 2024-07-15T07:03:27.457896+0000 mgr.y (mgr.25244) 274 : cephadm [INF] Deploying daemon osd.3 on smithi089 2024-07-15T07:03:28.822 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:28 smithi089 bash[79810]: cluster 2024-07-15T07:03:26.852713+0000 mgr.y (mgr.25244) 272 : cluster [DBG] pgmap v213: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:03:28.823 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:28 smithi089 bash[79810]: cephadm 2024-07-15T07:03:27.442824+0000 mgr.y (mgr.25244) 273 : cephadm [INF] Upgrade: Updating osd.3 2024-07-15T07:03:28.823 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:28 smithi089 bash[79810]: audit 2024-07-15T07:03:27.451469+0000 mon.a (mon.0) 465 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:03:28.823 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:28 smithi089 bash[79810]: audit 2024-07-15T07:03:27.452928+0000 mon.a (mon.0) 466 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-07-15T07:03:28.823 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:28 smithi089 bash[79810]: audit 2024-07-15T07:03:27.453899+0000 mon.a (mon.0) 467 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T07:03:28.823 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:28 smithi089 bash[79810]: cephadm 2024-07-15T07:03:27.457896+0000 mgr.y (mgr.25244) 274 : cephadm [INF] Deploying daemon osd.3 on smithi089 2024-07-15T07:03:28.823 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:28 smithi089 bash[82202]: cluster 2024-07-15T07:03:26.852713+0000 mgr.y (mgr.25244) 272 : cluster [DBG] pgmap v213: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:03:28.823 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:28 smithi089 bash[82202]: cephadm 2024-07-15T07:03:27.442824+0000 mgr.y (mgr.25244) 273 : cephadm [INF] Upgrade: Updating osd.3 2024-07-15T07:03:28.823 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:28 smithi089 bash[82202]: audit 2024-07-15T07:03:27.451469+0000 mon.a (mon.0) 465 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:03:28.823 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:28 smithi089 bash[82202]: audit 2024-07-15T07:03:27.452928+0000 mon.a (mon.0) 466 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-07-15T07:03:28.823 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:28 smithi089 bash[82202]: audit 2024-07-15T07:03:27.453899+0000 mon.a (mon.0) 467 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T07:03:28.824 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:28 smithi089 bash[82202]: cephadm 2024-07-15T07:03:27.457896+0000 mgr.y (mgr.25244) 274 : cephadm [INF] Deploying daemon osd.3 on smithi089 2024-07-15T07:03:29.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:29 smithi134 bash[67624]: audit 2024-07-15T07:03:28.493344+0000 mon.a (mon.0) 468 : audit [DBG] from='client.? 172.21.15.89:0/1137475569' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-15T07:03:29.855 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:29 smithi089 bash[82202]: audit 2024-07-15T07:03:28.493344+0000 mon.a (mon.0) 468 : audit [DBG] from='client.? 172.21.15.89:0/1137475569' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-15T07:03:29.855 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:29 smithi089 bash[79810]: audit 2024-07-15T07:03:28.493344+0000 mon.a (mon.0) 468 : audit [DBG] from='client.? 172.21.15.89:0/1137475569' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-15T07:03:30.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:30 smithi134 bash[67624]: cluster 2024-07-15T07:03:28.853282+0000 mgr.y (mgr.25244) 275 : cluster [DBG] pgmap v214: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:03:30.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:30 smithi134 bash[67624]: audit 2024-07-15T07:03:29.950226+0000 mon.a (mon.0) 469 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:03:30.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:30 smithi089 bash[79810]: cluster 2024-07-15T07:03:28.853282+0000 mgr.y (mgr.25244) 275 : cluster [DBG] pgmap v214: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:03:30.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:30 smithi089 bash[79810]: audit 2024-07-15T07:03:29.950226+0000 mon.a (mon.0) 469 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:03:30.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:30 smithi089 bash[82202]: cluster 2024-07-15T07:03:28.853282+0000 mgr.y (mgr.25244) 275 : cluster [DBG] pgmap v214: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:03:30.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:30 smithi089 bash[82202]: audit 2024-07-15T07:03:29.950226+0000 mon.a (mon.0) 469 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:03:31.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:31 smithi134 bash[67624]: audit 2024-07-15T07:03:30.800887+0000 mon.a (mon.0) 470 : audit [DBG] from='client.? 172.21.15.89:0/4062632789' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-15T07:03:31.870 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:31 smithi089 bash[82202]: audit 2024-07-15T07:03:30.800887+0000 mon.a (mon.0) 470 : audit [DBG] from='client.? 172.21.15.89:0/4062632789' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-15T07:03:31.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:31 smithi089 bash[79810]: audit 2024-07-15T07:03:30.800887+0000 mon.a (mon.0) 470 : audit [DBG] from='client.? 172.21.15.89:0/4062632789' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-15T07:03:32.357 INFO:journalctl@ceph.osd.1.smithi089.stdout:Jul 15 07:03:32 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:03:32.357 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:32 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:03:32.357 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 07:03:32 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:03:32.358 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:32 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:03:32.358 INFO:journalctl@ceph.osd.2.smithi089.stdout:Jul 15 07:03:32 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:03:32.358 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 07:03:32 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:03:32.359 INFO:journalctl@ceph.osd.0.smithi089.stdout:Jul 15 07:03:32 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:03:32.359 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 07:03:32 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:03:32.359 INFO:journalctl@ceph.osd.3.smithi089.stdout:Jul 15 07:03:32 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:03:32.359 INFO:journalctl@ceph.osd.3.smithi089.stdout:Jul 15 07:03:32 smithi089 systemd[1]: Stopping Ceph osd.3 for 75ba02bc-4274-11ef-bcad-c7b262605968... 2024-07-15T07:03:32.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:32 smithi089 bash[79810]: cluster 2024-07-15T07:03:30.854157+0000 mgr.y (mgr.25244) 276 : cluster [DBG] pgmap v215: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:03:32.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:32 smithi089 bash[82202]: cluster 2024-07-15T07:03:30.854157+0000 mgr.y (mgr.25244) 276 : cluster [DBG] pgmap v215: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:03:32.621 INFO:journalctl@ceph.osd.3.smithi089.stdout:Jul 15 07:03:32 smithi089 bash[51277]: debug 2024-07-15T07:03:32.473+0000 7fdee1e23640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-07-15T07:03:32.621 INFO:journalctl@ceph.osd.3.smithi089.stdout:Jul 15 07:03:32 smithi089 bash[51277]: debug 2024-07-15T07:03:32.473+0000 7fdee1e23640 -1 osd.3 98 *** Got signal Terminated *** 2024-07-15T07:03:32.621 INFO:journalctl@ceph.osd.3.smithi089.stdout:Jul 15 07:03:32 smithi089 bash[51277]: debug 2024-07-15T07:03:32.473+0000 7fdee1e23640 -1 osd.3 98 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-07-15T07:03:32.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:32 smithi134 bash[67624]: cluster 2024-07-15T07:03:30.854157+0000 mgr.y (mgr.25244) 276 : cluster [DBG] pgmap v215: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:03:32.805 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:03:32 smithi134 bash[60729]: ts=2024-07-15T07:03:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T07:03:33.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:33 smithi134 bash[67624]: cluster 2024-07-15T07:03:32.477448+0000 mon.a (mon.0) 471 : cluster [INF] osd.3 marked itself down and dead 2024-07-15T07:03:33.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:33 smithi134 bash[67624]: audit 2024-07-15T07:03:33.101877+0000 mon.c (mon.2) 9 : audit [DBG] from='client.? 172.21.15.89:0/3608180089' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-15T07:03:33.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:33 smithi089 bash[79810]: debug 2024-07-15T07:03:33.469+0000 7fb563a0a640 -1 log_channel(cluster) log [ERR] : Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-07-15T07:03:33.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:33 smithi089 bash[79810]: cluster 2024-07-15T07:03:32.477448+0000 mon.a (mon.0) 471 : cluster [INF] osd.3 marked itself down and dead 2024-07-15T07:03:33.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:33 smithi089 bash[79810]: audit 2024-07-15T07:03:33.101877+0000 mon.c (mon.2) 9 : audit [DBG] from='client.? 172.21.15.89:0/3608180089' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-15T07:03:33.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:33 smithi089 bash[82202]: cluster 2024-07-15T07:03:32.477448+0000 mon.a (mon.0) 471 : cluster [INF] osd.3 marked itself down and dead 2024-07-15T07:03:33.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:33 smithi089 bash[82202]: audit 2024-07-15T07:03:33.101877+0000 mon.c (mon.2) 9 : audit [DBG] from='client.? 172.21.15.89:0/3608180089' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-15T07:03:34.271 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 07:03:34 smithi089 bash[72894]: ::ffff:172.21.15.134 - - [15/Jul/2024:07:03:34] "GET /metrics HTTP/1.1" 200 33833 "" "Prometheus/2.43.0" 2024-07-15T07:03:34.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:34 smithi089 bash[79810]: cluster 2024-07-15T07:03:32.854863+0000 mgr.y (mgr.25244) 277 : cluster [DBG] pgmap v216: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:03:34.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:34 smithi089 bash[79810]: cluster 2024-07-15T07:03:33.473919+0000 mon.a (mon.0) 472 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-07-15T07:03:34.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:34 smithi089 bash[79810]: cluster 2024-07-15T07:03:33.473965+0000 mon.a (mon.0) 473 : cluster [ERR] Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-07-15T07:03:34.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:34 smithi089 bash[79810]: cluster 2024-07-15T07:03:33.490220+0000 mon.a (mon.0) 474 : cluster [DBG] osdmap e99: 8 total, 7 up, 8 in 2024-07-15T07:03:34.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:34 smithi089 bash[82202]: cluster 2024-07-15T07:03:32.854863+0000 mgr.y (mgr.25244) 277 : cluster [DBG] pgmap v216: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:03:34.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:34 smithi089 bash[82202]: cluster 2024-07-15T07:03:33.473919+0000 mon.a (mon.0) 472 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-07-15T07:03:34.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:34 smithi089 bash[82202]: cluster 2024-07-15T07:03:33.473965+0000 mon.a (mon.0) 473 : cluster [ERR] Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-07-15T07:03:34.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:34 smithi089 bash[82202]: cluster 2024-07-15T07:03:33.490220+0000 mon.a (mon.0) 474 : cluster [DBG] osdmap e99: 8 total, 7 up, 8 in 2024-07-15T07:03:34.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:34 smithi134 bash[67624]: cluster 2024-07-15T07:03:32.854863+0000 mgr.y (mgr.25244) 277 : cluster [DBG] pgmap v216: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:03:34.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:34 smithi134 bash[67624]: cluster 2024-07-15T07:03:33.473919+0000 mon.a (mon.0) 472 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-07-15T07:03:34.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:34 smithi134 bash[67624]: cluster 2024-07-15T07:03:33.473965+0000 mon.a (mon.0) 473 : cluster [ERR] Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-07-15T07:03:34.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:34 smithi134 bash[67624]: cluster 2024-07-15T07:03:33.490220+0000 mon.a (mon.0) 474 : cluster [DBG] osdmap e99: 8 total, 7 up, 8 in 2024-07-15T07:03:35.044 INFO:journalctl@ceph.osd.3.smithi089.stdout:Jul 15 07:03:34 smithi089 bash[102170]: ceph-75ba02bc-4274-11ef-bcad-c7b262605968-osd-3 2024-07-15T07:03:35.044 INFO:journalctl@ceph.osd.3.smithi089.stdout:Jul 15 07:03:34 smithi089 bash[102260]: Error response from daemon: No such container: ceph-75ba02bc-4274-11ef-bcad-c7b262605968-osd-3 2024-07-15T07:03:35.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:35 smithi134 bash[67624]: cluster 2024-07-15T07:03:34.491414+0000 mon.a (mon.0) 475 : cluster [DBG] osdmap e100: 8 total, 7 up, 8 in 2024-07-15T07:03:35.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:35 smithi134 bash[67624]: audit 2024-07-15T07:03:34.798225+0000 mon.a (mon.0) 476 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:03:35.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:35 smithi134 bash[67624]: audit 2024-07-15T07:03:34.799772+0000 mon.a (mon.0) 477 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:03:35.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:35 smithi134 bash[67624]: audit 2024-07-15T07:03:35.404431+0000 mon.a (mon.0) 478 : audit [DBG] from='client.? 172.21.15.89:0/3933737010' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-15T07:03:35.870 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:35 smithi089 bash[82202]: cluster 2024-07-15T07:03:34.491414+0000 mon.a (mon.0) 475 : cluster [DBG] osdmap e100: 8 total, 7 up, 8 in 2024-07-15T07:03:35.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:35 smithi089 bash[82202]: audit 2024-07-15T07:03:34.798225+0000 mon.a (mon.0) 476 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:03:35.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:35 smithi089 bash[82202]: audit 2024-07-15T07:03:34.799772+0000 mon.a (mon.0) 477 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:03:35.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:35 smithi089 bash[82202]: audit 2024-07-15T07:03:35.404431+0000 mon.a (mon.0) 478 : audit [DBG] from='client.? 172.21.15.89:0/3933737010' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-15T07:03:35.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:35 smithi089 bash[79810]: cluster 2024-07-15T07:03:34.491414+0000 mon.a (mon.0) 475 : cluster [DBG] osdmap e100: 8 total, 7 up, 8 in 2024-07-15T07:03:35.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:35 smithi089 bash[79810]: audit 2024-07-15T07:03:34.798225+0000 mon.a (mon.0) 476 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:03:35.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:35 smithi089 bash[79810]: audit 2024-07-15T07:03:34.799772+0000 mon.a (mon.0) 477 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:03:35.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:35 smithi089 bash[79810]: audit 2024-07-15T07:03:35.404431+0000 mon.a (mon.0) 478 : audit [DBG] from='client.? 172.21.15.89:0/3933737010' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-15T07:03:37.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:36 smithi134 bash[67624]: cluster 2024-07-15T07:03:34.855500+0000 mgr.y (mgr.25244) 278 : cluster [DBG] pgmap v219: 73 pgs: 11 stale+active+clean, 62 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:03:37.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:36 smithi089 bash[79810]: cluster 2024-07-15T07:03:34.855500+0000 mgr.y (mgr.25244) 278 : cluster [DBG] pgmap v219: 73 pgs: 11 stale+active+clean, 62 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:03:37.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:36 smithi089 bash[82202]: cluster 2024-07-15T07:03:34.855500+0000 mgr.y (mgr.25244) 278 : cluster [DBG] pgmap v219: 73 pgs: 11 stale+active+clean, 62 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:03:37.420 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 07:03:37 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:03:37.420 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:37 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:03:37.420 INFO:journalctl@ceph.osd.1.smithi089.stdout:Jul 15 07:03:37 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:03:37.421 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:37 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:03:37.421 INFO:journalctl@ceph.osd.0.smithi089.stdout:Jul 15 07:03:37 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:03:37.421 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 07:03:37 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:03:37.421 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 07:03:37 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:03:37.422 INFO:journalctl@ceph.osd.2.smithi089.stdout:Jul 15 07:03:37 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:03:37.422 INFO:journalctl@ceph.osd.3.smithi089.stdout:Jul 15 07:03:37 smithi089 systemd[1]: ceph-75ba02bc-4274-11ef-bcad-c7b262605968@osd.3.service: Deactivated successfully. 2024-07-15T07:03:37.422 INFO:journalctl@ceph.osd.3.smithi089.stdout:Jul 15 07:03:37 smithi089 systemd[1]: Stopped Ceph osd.3 for 75ba02bc-4274-11ef-bcad-c7b262605968. 2024-07-15T07:03:37.422 INFO:journalctl@ceph.osd.3.smithi089.stdout:Jul 15 07:03:37 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:03:37.422 INFO:journalctl@ceph.osd.3.smithi089.stdout:Jul 15 07:03:37 smithi089 systemd[1]: Started Ceph osd.3 for 75ba02bc-4274-11ef-bcad-c7b262605968. 2024-07-15T07:03:38.724 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:38 smithi089 bash[79810]: cluster 2024-07-15T07:03:36.856582+0000 mgr.y (mgr.25244) 279 : cluster [DBG] pgmap v220: 73 pgs: 19 active+undersized, 2 stale+active+clean, 2 active+undersized+degraded, 50 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2024-07-15T07:03:38.725 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:38 smithi089 bash[79810]: audit 2024-07-15T07:03:37.458031+0000 mon.a (mon.0) 479 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:03:38.725 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:38 smithi089 bash[79810]: audit 2024-07-15T07:03:37.463550+0000 mon.a (mon.0) 480 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:03:38.725 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:38 smithi089 bash[79810]: audit 2024-07-15T07:03:37.464554+0000 mon.a (mon.0) 481 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T07:03:38.725 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:38 smithi089 bash[79810]: audit 2024-07-15T07:03:37.699859+0000 mon.c (mon.2) 10 : audit [DBG] from='client.? 172.21.15.89:0/3730105815' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-15T07:03:38.725 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:38 smithi089 bash[79810]: cluster 2024-07-15T07:03:37.798821+0000 mon.a (mon.0) 482 : cluster [WRN] Health check failed: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded (PG_DEGRADED) 2024-07-15T07:03:38.725 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:38 smithi089 bash[82202]: cluster 2024-07-15T07:03:36.856582+0000 mgr.y (mgr.25244) 279 : cluster [DBG] pgmap v220: 73 pgs: 19 active+undersized, 2 stale+active+clean, 2 active+undersized+degraded, 50 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2024-07-15T07:03:38.725 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:38 smithi089 bash[82202]: audit 2024-07-15T07:03:37.458031+0000 mon.a (mon.0) 479 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:03:38.726 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:38 smithi089 bash[82202]: audit 2024-07-15T07:03:37.463550+0000 mon.a (mon.0) 480 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:03:38.726 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:38 smithi089 bash[82202]: audit 2024-07-15T07:03:37.464554+0000 mon.a (mon.0) 481 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T07:03:38.726 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:38 smithi089 bash[82202]: audit 2024-07-15T07:03:37.699859+0000 mon.c (mon.2) 10 : audit [DBG] from='client.? 172.21.15.89:0/3730105815' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-15T07:03:38.726 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:38 smithi089 bash[82202]: cluster 2024-07-15T07:03:37.798821+0000 mon.a (mon.0) 482 : cluster [WRN] Health check failed: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded (PG_DEGRADED) 2024-07-15T07:03:38.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:38 smithi134 bash[67624]: cluster 2024-07-15T07:03:36.856582+0000 mgr.y (mgr.25244) 279 : cluster [DBG] pgmap v220: 73 pgs: 19 active+undersized, 2 stale+active+clean, 2 active+undersized+degraded, 50 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2024-07-15T07:03:38.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:38 smithi134 bash[67624]: audit 2024-07-15T07:03:37.458031+0000 mon.a (mon.0) 479 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:03:38.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:38 smithi134 bash[67624]: audit 2024-07-15T07:03:37.463550+0000 mon.a (mon.0) 480 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:03:38.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:38 smithi134 bash[67624]: audit 2024-07-15T07:03:37.464554+0000 mon.a (mon.0) 481 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T07:03:38.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:38 smithi134 bash[67624]: audit 2024-07-15T07:03:37.699859+0000 mon.c (mon.2) 10 : audit [DBG] from='client.? 172.21.15.89:0/3730105815' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-15T07:03:38.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:38 smithi134 bash[67624]: cluster 2024-07-15T07:03:37.798821+0000 mon.a (mon.0) 482 : cluster [WRN] Health check failed: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded (PG_DEGRADED) 2024-07-15T07:03:39.371 INFO:journalctl@ceph.osd.3.smithi089.stdout:Jul 15 07:03:39 smithi089 bash[102436]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-07-15T07:03:39.371 INFO:journalctl@ceph.osd.3.smithi089.stdout:Jul 15 07:03:39 smithi089 bash[102436]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-07-15T07:03:39.371 INFO:journalctl@ceph.osd.3.smithi089.stdout:Jul 15 07:03:39 smithi089 bash[102436]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-07-15T07:03:39.371 INFO:journalctl@ceph.osd.3.smithi089.stdout:Jul 15 07:03:39 smithi089 bash[102436]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-07-15T07:03:39.371 INFO:journalctl@ceph.osd.3.smithi089.stdout:Jul 15 07:03:39 smithi089 bash[102436]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-07-15T07:03:39.371 INFO:journalctl@ceph.osd.3.smithi089.stdout:Jul 15 07:03:39 smithi089 bash[102436]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-07-15T07:03:39.372 INFO:journalctl@ceph.osd.3.smithi089.stdout:Jul 15 07:03:39 smithi089 bash[102436]: --> ceph-volume raw activate successful for osd ID: 3 2024-07-15T07:03:40.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:40 smithi134 bash[67624]: cluster 2024-07-15T07:03:38.857359+0000 mgr.y (mgr.25244) 280 : cluster [DBG] pgmap v221: 73 pgs: 24 active+undersized, 3 active+undersized+degraded, 46 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-07-15T07:03:40.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:40 smithi134 bash[67624]: audit 2024-07-15T07:03:39.996451+0000 mon.a (mon.0) 483 : audit [DBG] from='client.? 172.21.15.89:0/1103810561' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-15T07:03:40.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:40 smithi089 bash[79810]: cluster 2024-07-15T07:03:38.857359+0000 mgr.y (mgr.25244) 280 : cluster [DBG] pgmap v221: 73 pgs: 24 active+undersized, 3 active+undersized+degraded, 46 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-07-15T07:03:40.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:40 smithi089 bash[79810]: audit 2024-07-15T07:03:39.996451+0000 mon.a (mon.0) 483 : audit [DBG] from='client.? 172.21.15.89:0/1103810561' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-15T07:03:40.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:40 smithi089 bash[82202]: cluster 2024-07-15T07:03:38.857359+0000 mgr.y (mgr.25244) 280 : cluster [DBG] pgmap v221: 73 pgs: 24 active+undersized, 3 active+undersized+degraded, 46 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-07-15T07:03:40.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:40 smithi089 bash[82202]: audit 2024-07-15T07:03:39.996451+0000 mon.a (mon.0) 483 : audit [DBG] from='client.? 172.21.15.89:0/1103810561' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-15T07:03:41.370 INFO:journalctl@ceph.osd.3.smithi089.stdout:Jul 15 07:03:41 smithi089 bash[102706]: debug 2024-07-15T07:03:41.077+0000 7f46f5906640 1 -- 172.21.15.89:0/721209420 <== mon.2 v2:172.21.15.89:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5604f72ab1e0 con 0x5604f726b800 2024-07-15T07:03:42.120 INFO:journalctl@ceph.osd.3.smithi089.stdout:Jul 15 07:03:41 smithi089 bash[102706]: debug 2024-07-15T07:03:41.621+0000 7f46f816c640 -1 Falling back to public interface 2024-07-15T07:03:42.804 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:03:42 smithi134 bash[60729]: ts=2024-07-15T07:03:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T07:03:42.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:42 smithi134 bash[67624]: cluster 2024-07-15T07:03:40.858357+0000 mgr.y (mgr.25244) 281 : cluster [DBG] pgmap v222: 73 pgs: 24 active+undersized, 3 active+undersized+degraded, 46 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-07-15T07:03:42.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:42 smithi134 bash[67624]: audit 2024-07-15T07:03:42.297962+0000 mon.c (mon.2) 11 : audit [DBG] from='client.? 172.21.15.89:0/709774136' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-15T07:03:42.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:42 smithi089 bash[79810]: cluster 2024-07-15T07:03:40.858357+0000 mgr.y (mgr.25244) 281 : cluster [DBG] pgmap v222: 73 pgs: 24 active+undersized, 3 active+undersized+degraded, 46 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-07-15T07:03:42.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:42 smithi089 bash[79810]: audit 2024-07-15T07:03:42.297962+0000 mon.c (mon.2) 11 : audit [DBG] from='client.? 172.21.15.89:0/709774136' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-15T07:03:42.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:42 smithi089 bash[82202]: cluster 2024-07-15T07:03:40.858357+0000 mgr.y (mgr.25244) 281 : cluster [DBG] pgmap v222: 73 pgs: 24 active+undersized, 3 active+undersized+degraded, 46 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-07-15T07:03:42.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:42 smithi089 bash[82202]: audit 2024-07-15T07:03:42.297962+0000 mon.c (mon.2) 11 : audit [DBG] from='client.? 172.21.15.89:0/709774136' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-15T07:03:44.463 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 07:03:44 smithi089 bash[72894]: ::ffff:172.21.15.134 - - [15/Jul/2024:07:03:44] "GET /metrics HTTP/1.1" 200 33980 "" "Prometheus/2.43.0" 2024-07-15T07:03:44.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:44 smithi134 bash[67624]: cluster 2024-07-15T07:03:42.859185+0000 mgr.y (mgr.25244) 282 : cluster [DBG] pgmap v223: 73 pgs: 24 active+undersized, 3 active+undersized+degraded, 46 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-07-15T07:03:44.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:44 smithi134 bash[67624]: audit 2024-07-15T07:03:44.096934+0000 mon.a (mon.0) 484 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:03:44.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:44 smithi134 bash[67624]: audit 2024-07-15T07:03:44.104756+0000 mon.a (mon.0) 485 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:03:44.870 INFO:journalctl@ceph.osd.3.smithi089.stdout:Jul 15 07:03:44 smithi089 bash[102706]: debug 2024-07-15T07:03:44.677+0000 7f46f816c640 -1 osd.3 98 log_to_monitors true 2024-07-15T07:03:44.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:44 smithi089 bash[82202]: cluster 2024-07-15T07:03:42.859185+0000 mgr.y (mgr.25244) 282 : cluster [DBG] pgmap v223: 73 pgs: 24 active+undersized, 3 active+undersized+degraded, 46 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-07-15T07:03:44.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:44 smithi089 bash[82202]: audit 2024-07-15T07:03:44.096934+0000 mon.a (mon.0) 484 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:03:44.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:44 smithi089 bash[82202]: audit 2024-07-15T07:03:44.104756+0000 mon.a (mon.0) 485 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:03:44.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:44 smithi089 bash[79810]: cluster 2024-07-15T07:03:42.859185+0000 mgr.y (mgr.25244) 282 : cluster [DBG] pgmap v223: 73 pgs: 24 active+undersized, 3 active+undersized+degraded, 46 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-07-15T07:03:44.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:44 smithi089 bash[79810]: audit 2024-07-15T07:03:44.096934+0000 mon.a (mon.0) 484 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:03:44.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:44 smithi089 bash[79810]: audit 2024-07-15T07:03:44.104756+0000 mon.a (mon.0) 485 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:03:45.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:45 smithi134 bash[67624]: audit 2024-07-15T07:03:44.607661+0000 mon.a (mon.0) 486 : audit [DBG] from='client.? 172.21.15.89:0/1269215158' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-15T07:03:45.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:45 smithi134 bash[67624]: audit 2024-07-15T07:03:44.686030+0000 mon.c (mon.2) 12 : audit [INF] from='osd.3 [v2:172.21.15.89:6826/3416042041,v1:172.21.15.89:6827/3416042041]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-07-15T07:03:45.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:45 smithi134 bash[67624]: audit 2024-07-15T07:03:44.686365+0000 mon.a (mon.0) 487 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-07-15T07:03:45.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:45 smithi134 bash[67624]: audit 2024-07-15T07:03:44.805512+0000 mon.a (mon.0) 488 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:03:45.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:45 smithi134 bash[67624]: audit 2024-07-15T07:03:44.811685+0000 mon.a (mon.0) 489 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:03:45.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:45 smithi089 bash[79810]: audit 2024-07-15T07:03:44.607661+0000 mon.a (mon.0) 486 : audit [DBG] from='client.? 172.21.15.89:0/1269215158' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-15T07:03:45.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:45 smithi089 bash[79810]: audit 2024-07-15T07:03:44.686030+0000 mon.c (mon.2) 12 : audit [INF] from='osd.3 [v2:172.21.15.89:6826/3416042041,v1:172.21.15.89:6827/3416042041]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-07-15T07:03:45.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:45 smithi089 bash[79810]: audit 2024-07-15T07:03:44.686365+0000 mon.a (mon.0) 487 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-07-15T07:03:45.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:45 smithi089 bash[79810]: audit 2024-07-15T07:03:44.805512+0000 mon.a (mon.0) 488 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:03:45.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:45 smithi089 bash[79810]: audit 2024-07-15T07:03:44.811685+0000 mon.a (mon.0) 489 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:03:45.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:45 smithi089 bash[82202]: audit 2024-07-15T07:03:44.607661+0000 mon.a (mon.0) 486 : audit [DBG] from='client.? 172.21.15.89:0/1269215158' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-15T07:03:45.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:45 smithi089 bash[82202]: audit 2024-07-15T07:03:44.686030+0000 mon.c (mon.2) 12 : audit [INF] from='osd.3 [v2:172.21.15.89:6826/3416042041,v1:172.21.15.89:6827/3416042041]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-07-15T07:03:45.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:45 smithi089 bash[82202]: audit 2024-07-15T07:03:44.686365+0000 mon.a (mon.0) 487 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-07-15T07:03:45.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:45 smithi089 bash[82202]: audit 2024-07-15T07:03:44.805512+0000 mon.a (mon.0) 488 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:03:45.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:45 smithi089 bash[82202]: audit 2024-07-15T07:03:44.811685+0000 mon.a (mon.0) 489 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:03:46.371 INFO:journalctl@ceph.osd.3.smithi089.stdout:Jul 15 07:03:46 smithi089 bash[102706]: debug 2024-07-15T07:03:46.009+0000 7f46ebf0e640 -1 osd.3 98 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-07-15T07:03:46.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:46 smithi134 bash[67624]: cluster 2024-07-15T07:03:44.859685+0000 mgr.y (mgr.25244) 283 : cluster [DBG] pgmap v224: 73 pgs: 24 active+undersized, 3 active+undersized+degraded, 46 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-07-15T07:03:46.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:46 smithi134 bash[67624]: audit 2024-07-15T07:03:45.479592+0000 mon.a (mon.0) 490 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-07-15T07:03:46.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:46 smithi134 bash[67624]: cluster 2024-07-15T07:03:45.479792+0000 mon.a (mon.0) 491 : cluster [DBG] osdmap e101: 8 total, 7 up, 8 in 2024-07-15T07:03:46.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:46 smithi134 bash[67624]: audit 2024-07-15T07:03:45.483925+0000 mon.c (mon.2) 13 : audit [INF] from='osd.3 [v2:172.21.15.89:6826/3416042041,v1:172.21.15.89:6827/3416042041]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2024-07-15T07:03:46.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:46 smithi134 bash[67624]: audit 2024-07-15T07:03:45.492888+0000 mon.a (mon.0) 492 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2024-07-15T07:03:46.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:46 smithi134 bash[67624]: audit 2024-07-15T07:03:45.999675+0000 mon.a (mon.0) 493 : audit [INF] from='osd.3 ' entity='osd.3' 2024-07-15T07:03:46.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:46 smithi089 bash[79810]: cluster 2024-07-15T07:03:44.859685+0000 mgr.y (mgr.25244) 283 : cluster [DBG] pgmap v224: 73 pgs: 24 active+undersized, 3 active+undersized+degraded, 46 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-07-15T07:03:46.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:46 smithi089 bash[79810]: audit 2024-07-15T07:03:45.479592+0000 mon.a (mon.0) 490 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-07-15T07:03:46.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:46 smithi089 bash[79810]: cluster 2024-07-15T07:03:45.479792+0000 mon.a (mon.0) 491 : cluster [DBG] osdmap e101: 8 total, 7 up, 8 in 2024-07-15T07:03:46.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:46 smithi089 bash[79810]: audit 2024-07-15T07:03:45.483925+0000 mon.c (mon.2) 13 : audit [INF] from='osd.3 [v2:172.21.15.89:6826/3416042041,v1:172.21.15.89:6827/3416042041]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2024-07-15T07:03:46.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:46 smithi089 bash[79810]: audit 2024-07-15T07:03:45.492888+0000 mon.a (mon.0) 492 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2024-07-15T07:03:46.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:46 smithi089 bash[79810]: audit 2024-07-15T07:03:45.999675+0000 mon.a (mon.0) 493 : audit [INF] from='osd.3 ' entity='osd.3' 2024-07-15T07:03:46.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:46 smithi089 bash[82202]: cluster 2024-07-15T07:03:44.859685+0000 mgr.y (mgr.25244) 283 : cluster [DBG] pgmap v224: 73 pgs: 24 active+undersized, 3 active+undersized+degraded, 46 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-07-15T07:03:46.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:46 smithi089 bash[82202]: audit 2024-07-15T07:03:45.479592+0000 mon.a (mon.0) 490 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-07-15T07:03:46.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:46 smithi089 bash[82202]: cluster 2024-07-15T07:03:45.479792+0000 mon.a (mon.0) 491 : cluster [DBG] osdmap e101: 8 total, 7 up, 8 in 2024-07-15T07:03:46.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:46 smithi089 bash[82202]: audit 2024-07-15T07:03:45.483925+0000 mon.c (mon.2) 13 : audit [INF] from='osd.3 [v2:172.21.15.89:6826/3416042041,v1:172.21.15.89:6827/3416042041]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2024-07-15T07:03:46.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:46 smithi089 bash[82202]: audit 2024-07-15T07:03:45.492888+0000 mon.a (mon.0) 492 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2024-07-15T07:03:46.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:46 smithi089 bash[82202]: audit 2024-07-15T07:03:45.999675+0000 mon.a (mon.0) 493 : audit [INF] from='osd.3 ' entity='osd.3' 2024-07-15T07:03:47.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:46 smithi089 bash[79810]: debug 2024-07-15T07:03:46.993+0000 7fb563a0a640 -1 log_channel(cluster) log [ERR] : Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-07-15T07:03:47.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:47 smithi134 bash[67624]: audit 2024-07-15T07:03:46.910464+0000 mon.c (mon.2) 14 : audit [DBG] from='client.? 172.21.15.89:0/646110953' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-15T07:03:47.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:47 smithi134 bash[67624]: cluster 2024-07-15T07:03:46.997729+0000 mon.a (mon.0) 494 : cluster [ERR] Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-07-15T07:03:47.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:47 smithi134 bash[67624]: cluster 2024-07-15T07:03:46.997789+0000 mon.a (mon.0) 495 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-07-15T07:03:47.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:47 smithi134 bash[67624]: cluster 2024-07-15T07:03:47.010427+0000 mon.a (mon.0) 496 : cluster [INF] osd.3 [v2:172.21.15.89:6826/3416042041,v1:172.21.15.89:6827/3416042041] boot 2024-07-15T07:03:47.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:47 smithi134 bash[67624]: cluster 2024-07-15T07:03:47.010534+0000 mon.a (mon.0) 497 : cluster [DBG] osdmap e102: 8 total, 8 up, 8 in 2024-07-15T07:03:47.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:47 smithi134 bash[67624]: audit 2024-07-15T07:03:47.010953+0000 mon.a (mon.0) 498 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-07-15T07:03:47.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:47 smithi134 bash[67624]: cluster 2024-07-15T07:03:47.313224+0000 mon.a (mon.0) 499 : cluster [WRN] Health check update: Degraded data redundancy: 4/72 objects degraded (5.556%), 3 pgs degraded (PG_DEGRADED) 2024-07-15T07:03:47.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:47 smithi089 bash[79810]: audit 2024-07-15T07:03:46.910464+0000 mon.c (mon.2) 14 : audit [DBG] from='client.? 172.21.15.89:0/646110953' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-15T07:03:47.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:47 smithi089 bash[79810]: cluster 2024-07-15T07:03:46.997729+0000 mon.a (mon.0) 494 : cluster [ERR] Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-07-15T07:03:47.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:47 smithi089 bash[79810]: cluster 2024-07-15T07:03:46.997789+0000 mon.a (mon.0) 495 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-07-15T07:03:47.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:47 smithi089 bash[79810]: cluster 2024-07-15T07:03:47.010427+0000 mon.a (mon.0) 496 : cluster [INF] osd.3 [v2:172.21.15.89:6826/3416042041,v1:172.21.15.89:6827/3416042041] boot 2024-07-15T07:03:47.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:47 smithi089 bash[79810]: cluster 2024-07-15T07:03:47.010534+0000 mon.a (mon.0) 497 : cluster [DBG] osdmap e102: 8 total, 8 up, 8 in 2024-07-15T07:03:47.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:47 smithi089 bash[79810]: audit 2024-07-15T07:03:47.010953+0000 mon.a (mon.0) 498 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-07-15T07:03:47.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:47 smithi089 bash[79810]: cluster 2024-07-15T07:03:47.313224+0000 mon.a (mon.0) 499 : cluster [WRN] Health check update: Degraded data redundancy: 4/72 objects degraded (5.556%), 3 pgs degraded (PG_DEGRADED) 2024-07-15T07:03:47.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:47 smithi089 bash[82202]: audit 2024-07-15T07:03:46.910464+0000 mon.c (mon.2) 14 : audit [DBG] from='client.? 172.21.15.89:0/646110953' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-15T07:03:47.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:47 smithi089 bash[82202]: cluster 2024-07-15T07:03:46.997729+0000 mon.a (mon.0) 494 : cluster [ERR] Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-07-15T07:03:47.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:47 smithi089 bash[82202]: cluster 2024-07-15T07:03:46.997789+0000 mon.a (mon.0) 495 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-07-15T07:03:47.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:47 smithi089 bash[82202]: cluster 2024-07-15T07:03:47.010427+0000 mon.a (mon.0) 496 : cluster [INF] osd.3 [v2:172.21.15.89:6826/3416042041,v1:172.21.15.89:6827/3416042041] boot 2024-07-15T07:03:47.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:47 smithi089 bash[82202]: cluster 2024-07-15T07:03:47.010534+0000 mon.a (mon.0) 497 : cluster [DBG] osdmap e102: 8 total, 8 up, 8 in 2024-07-15T07:03:47.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:47 smithi089 bash[82202]: audit 2024-07-15T07:03:47.010953+0000 mon.a (mon.0) 498 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-07-15T07:03:47.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:47 smithi089 bash[82202]: cluster 2024-07-15T07:03:47.313224+0000 mon.a (mon.0) 499 : cluster [WRN] Health check update: Degraded data redundancy: 4/72 objects degraded (5.556%), 3 pgs degraded (PG_DEGRADED) 2024-07-15T07:03:48.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:48 smithi134 bash[67624]: cluster 2024-07-15T07:03:46.860730+0000 mgr.y (mgr.25244) 284 : cluster [DBG] pgmap v226: 73 pgs: 24 active+undersized, 3 active+undersized+degraded, 46 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-07-15T07:03:48.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:48 smithi134 bash[67624]: cluster 2024-07-15T07:03:48.012867+0000 mon.a (mon.0) 500 : cluster [DBG] osdmap e103: 8 total, 8 up, 8 in 2024-07-15T07:03:48.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:48 smithi089 bash[79810]: cluster 2024-07-15T07:03:46.860730+0000 mgr.y (mgr.25244) 284 : cluster [DBG] pgmap v226: 73 pgs: 24 active+undersized, 3 active+undersized+degraded, 46 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-07-15T07:03:48.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:48 smithi089 bash[79810]: cluster 2024-07-15T07:03:48.012867+0000 mon.a (mon.0) 500 : cluster [DBG] osdmap e103: 8 total, 8 up, 8 in 2024-07-15T07:03:48.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:48 smithi089 bash[82202]: cluster 2024-07-15T07:03:46.860730+0000 mgr.y (mgr.25244) 284 : cluster [DBG] pgmap v226: 73 pgs: 24 active+undersized, 3 active+undersized+degraded, 46 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-07-15T07:03:48.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:48 smithi089 bash[82202]: cluster 2024-07-15T07:03:48.012867+0000 mon.a (mon.0) 500 : cluster [DBG] osdmap e103: 8 total, 8 up, 8 in 2024-07-15T07:03:49.228 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 18.2.2-1609-g6ea7aaa9 (6ea7aaa9f93d34dc4d21862afe00140dfddbd706) reef (stable)": 4 2024-07-15T07:03:49.804 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:49 smithi134 bash[67624]: audit 2024-07-15T07:03:49.219352+0000 mon.a (mon.0) 501 : audit [DBG] from='client.? 172.21.15.89:0/386600268' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-15T07:03:49.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:49 smithi089 bash[79810]: audit 2024-07-15T07:03:49.219352+0000 mon.a (mon.0) 501 : audit [DBG] from='client.? 172.21.15.89:0/386600268' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-15T07:03:49.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:49 smithi089 bash[82202]: audit 2024-07-15T07:03:49.219352+0000 mon.a (mon.0) 501 : audit [DBG] from='client.? 172.21.15.89:0/386600268' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-15T07:03:50.473 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -e sha1=6ea7aaa9f93d34dc4d21862afe00140dfddbd706 -- bash -c 'ceph orch upgrade pause' 2024-07-15T07:03:51.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:50 smithi134 bash[67624]: cluster 2024-07-15T07:03:48.861562+0000 mgr.y (mgr.25244) 285 : cluster [DBG] pgmap v229: 73 pgs: 20 active+undersized, 3 active+undersized+degraded, 50 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-07-15T07:03:51.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:50 smithi134 bash[67624]: audit 2024-07-15T07:03:49.799985+0000 mon.a (mon.0) 502 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:03:51.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:50 smithi134 bash[67624]: audit 2024-07-15T07:03:49.821409+0000 mon.a (mon.0) 503 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:03:51.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:50 smithi089 bash[79810]: cluster 2024-07-15T07:03:48.861562+0000 mgr.y (mgr.25244) 285 : cluster [DBG] pgmap v229: 73 pgs: 20 active+undersized, 3 active+undersized+degraded, 50 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-07-15T07:03:51.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:50 smithi089 bash[79810]: audit 2024-07-15T07:03:49.799985+0000 mon.a (mon.0) 502 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:03:51.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:50 smithi089 bash[79810]: audit 2024-07-15T07:03:49.821409+0000 mon.a (mon.0) 503 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:03:51.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:50 smithi089 bash[82202]: cluster 2024-07-15T07:03:48.861562+0000 mgr.y (mgr.25244) 285 : cluster [DBG] pgmap v229: 73 pgs: 20 active+undersized, 3 active+undersized+degraded, 50 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-07-15T07:03:51.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:50 smithi089 bash[82202]: audit 2024-07-15T07:03:49.799985+0000 mon.a (mon.0) 502 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:03:51.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:50 smithi089 bash[82202]: audit 2024-07-15T07:03:49.821409+0000 mon.a (mon.0) 503 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:03:51.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:51 smithi089 bash[79810]: cluster 2024-07-15T07:03:51.797251+0000 mon.a (mon.0) 504 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/72 objects degraded (5.556%), 3 pgs degraded) 2024-07-15T07:03:51.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:51 smithi089 bash[82202]: cluster 2024-07-15T07:03:51.797251+0000 mon.a (mon.0) 504 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/72 objects degraded (5.556%), 3 pgs degraded) 2024-07-15T07:03:52.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:51 smithi134 bash[67624]: cluster 2024-07-15T07:03:51.797251+0000 mon.a (mon.0) 504 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/72 objects degraded (5.556%), 3 pgs degraded) 2024-07-15T07:03:52.804 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:03:52 smithi134 bash[60729]: ts=2024-07-15T07:03:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T07:03:53.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:52 smithi089 bash[79810]: cluster 2024-07-15T07:03:50.862572+0000 mgr.y (mgr.25244) 286 : cluster [DBG] pgmap v230: 73 pgs: 1 active+undersized, 72 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:03:53.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:52 smithi089 bash[82202]: cluster 2024-07-15T07:03:50.862572+0000 mgr.y (mgr.25244) 286 : cluster [DBG] pgmap v230: 73 pgs: 1 active+undersized, 72 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:03:53.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:52 smithi134 bash[67624]: cluster 2024-07-15T07:03:50.862572+0000 mgr.y (mgr.25244) 286 : cluster [DBG] pgmap v230: 73 pgs: 1 active+undersized, 72 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:03:54.159 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:53 smithi089 bash[79810]: cluster 2024-07-15T07:03:52.863251+0000 mgr.y (mgr.25244) 287 : cluster [DBG] pgmap v231: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:03:54.160 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:53 smithi089 bash[82202]: cluster 2024-07-15T07:03:52.863251+0000 mgr.y (mgr.25244) 287 : cluster [DBG] pgmap v231: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:03:54.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:53 smithi134 bash[67624]: cluster 2024-07-15T07:03:52.863251+0000 mgr.y (mgr.25244) 287 : cluster [DBG] pgmap v231: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:03:54.428 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 07:03:54 smithi089 bash[72894]: ::ffff:172.21.15.134 - - [15/Jul/2024:07:03:54] "GET /metrics HTTP/1.1" 200 33836 "" "Prometheus/2.43.0" 2024-07-15T07:03:55.615 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T07:03:56.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:55 smithi134 bash[67624]: cluster 2024-07-15T07:03:54.863981+0000 mgr.y (mgr.25244) 288 : cluster [DBG] pgmap v232: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:03:56.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:55 smithi134 bash[67624]: audit 2024-07-15T07:03:54.969607+0000 mon.a (mon.0) 505 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:03:56.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:55 smithi134 bash[67624]: audit 2024-07-15T07:03:55.156352+0000 mon.a (mon.0) 506 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:03:56.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:55 smithi134 bash[67624]: audit 2024-07-15T07:03:55.164717+0000 mon.a (mon.0) 507 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:03:56.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:55 smithi134 bash[67624]: audit 2024-07-15T07:03:55.166239+0000 mon.a (mon.0) 508 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T07:03:56.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:55 smithi134 bash[67624]: audit 2024-07-15T07:03:55.167651+0000 mon.a (mon.0) 509 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T07:03:56.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:55 smithi134 bash[67624]: audit 2024-07-15T07:03:55.175660+0000 mon.a (mon.0) 510 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:03:56.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:55 smithi134 bash[67624]: audit 2024-07-15T07:03:55.309245+0000 mon.a (mon.0) 511 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T07:03:56.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:55 smithi134 bash[67624]: audit 2024-07-15T07:03:55.313716+0000 mon.a (mon.0) 512 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-15T07:03:56.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:55 smithi134 bash[67624]: audit 2024-07-15T07:03:55.316261+0000 mon.a (mon.0) 513 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-15T07:03:56.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:55 smithi134 bash[67624]: audit 2024-07-15T07:03:55.318286+0000 mon.a (mon.0) 514 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-15T07:03:56.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:55 smithi134 bash[67624]: audit 2024-07-15T07:03:55.320701+0000 mon.a (mon.0) 515 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-07-15T07:03:56.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:55 smithi134 bash[67624]: audit 2024-07-15T07:03:55.321063+0000 mgr.y (mgr.25244) 289 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-07-15T07:03:56.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:55 smithi134 bash[67624]: cephadm 2024-07-15T07:03:55.322540+0000 mgr.y (mgr.25244) 290 : cephadm [INF] Upgrade: osd.4 is safe to restart 2024-07-15T07:03:56.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:55 smithi089 bash[79810]: cluster 2024-07-15T07:03:54.863981+0000 mgr.y (mgr.25244) 288 : cluster [DBG] pgmap v232: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:03:56.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:55 smithi089 bash[79810]: audit 2024-07-15T07:03:54.969607+0000 mon.a (mon.0) 505 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:03:56.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:55 smithi089 bash[79810]: audit 2024-07-15T07:03:55.156352+0000 mon.a (mon.0) 506 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:03:56.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:55 smithi089 bash[79810]: audit 2024-07-15T07:03:55.164717+0000 mon.a (mon.0) 507 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:03:56.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:55 smithi089 bash[79810]: audit 2024-07-15T07:03:55.166239+0000 mon.a (mon.0) 508 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T07:03:56.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:55 smithi089 bash[79810]: audit 2024-07-15T07:03:55.167651+0000 mon.a (mon.0) 509 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T07:03:56.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:55 smithi089 bash[79810]: audit 2024-07-15T07:03:55.175660+0000 mon.a (mon.0) 510 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:03:56.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:55 smithi089 bash[79810]: audit 2024-07-15T07:03:55.309245+0000 mon.a (mon.0) 511 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T07:03:56.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:55 smithi089 bash[79810]: audit 2024-07-15T07:03:55.313716+0000 mon.a (mon.0) 512 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-15T07:03:56.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:55 smithi089 bash[79810]: audit 2024-07-15T07:03:55.316261+0000 mon.a (mon.0) 513 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-15T07:03:56.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:55 smithi089 bash[79810]: audit 2024-07-15T07:03:55.318286+0000 mon.a (mon.0) 514 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-15T07:03:56.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:55 smithi089 bash[79810]: audit 2024-07-15T07:03:55.320701+0000 mon.a (mon.0) 515 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-07-15T07:03:56.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:55 smithi089 bash[79810]: audit 2024-07-15T07:03:55.321063+0000 mgr.y (mgr.25244) 289 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-07-15T07:03:56.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:55 smithi089 bash[79810]: cephadm 2024-07-15T07:03:55.322540+0000 mgr.y (mgr.25244) 290 : cephadm [INF] Upgrade: osd.4 is safe to restart 2024-07-15T07:03:56.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:55 smithi089 bash[82202]: cluster 2024-07-15T07:03:54.863981+0000 mgr.y (mgr.25244) 288 : cluster [DBG] pgmap v232: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:03:56.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:55 smithi089 bash[82202]: audit 2024-07-15T07:03:54.969607+0000 mon.a (mon.0) 505 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:03:56.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:55 smithi089 bash[82202]: audit 2024-07-15T07:03:55.156352+0000 mon.a (mon.0) 506 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:03:56.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:55 smithi089 bash[82202]: audit 2024-07-15T07:03:55.164717+0000 mon.a (mon.0) 507 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:03:56.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:55 smithi089 bash[82202]: audit 2024-07-15T07:03:55.166239+0000 mon.a (mon.0) 508 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T07:03:56.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:55 smithi089 bash[82202]: audit 2024-07-15T07:03:55.167651+0000 mon.a (mon.0) 509 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T07:03:56.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:55 smithi089 bash[82202]: audit 2024-07-15T07:03:55.175660+0000 mon.a (mon.0) 510 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:03:56.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:55 smithi089 bash[82202]: audit 2024-07-15T07:03:55.309245+0000 mon.a (mon.0) 511 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T07:03:56.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:55 smithi089 bash[82202]: audit 2024-07-15T07:03:55.313716+0000 mon.a (mon.0) 512 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-15T07:03:56.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:55 smithi089 bash[82202]: audit 2024-07-15T07:03:55.316261+0000 mon.a (mon.0) 513 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-15T07:03:56.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:55 smithi089 bash[82202]: audit 2024-07-15T07:03:55.318286+0000 mon.a (mon.0) 514 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-15T07:03:56.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:55 smithi089 bash[82202]: audit 2024-07-15T07:03:55.320701+0000 mon.a (mon.0) 515 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-07-15T07:03:56.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:55 smithi089 bash[82202]: audit 2024-07-15T07:03:55.321063+0000 mgr.y (mgr.25244) 289 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-07-15T07:03:56.374 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:55 smithi089 bash[82202]: cephadm 2024-07-15T07:03:55.322540+0000 mgr.y (mgr.25244) 290 : cephadm [INF] Upgrade: osd.4 is safe to restart 2024-07-15T07:03:59.011 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:58 smithi089 bash[79810]: cluster 2024-07-15T07:03:56.864941+0000 mgr.y (mgr.25244) 291 : cluster [DBG] pgmap v233: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:03:59.011 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:58 smithi089 bash[79810]: cephadm 2024-07-15T07:03:57.693741+0000 mgr.y (mgr.25244) 292 : cephadm [INF] Upgrade: Updating osd.4 2024-07-15T07:03:59.011 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:58 smithi089 bash[79810]: audit 2024-07-15T07:03:57.703522+0000 mon.a (mon.0) 516 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:03:59.011 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:58 smithi089 bash[79810]: audit 2024-07-15T07:03:57.705564+0000 mon.a (mon.0) 517 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-07-15T07:03:59.011 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:58 smithi089 bash[79810]: audit 2024-07-15T07:03:57.707047+0000 mon.a (mon.0) 518 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T07:03:59.011 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:03:58 smithi089 bash[79810]: cephadm 2024-07-15T07:03:57.711969+0000 mgr.y (mgr.25244) 293 : cephadm [INF] Deploying daemon osd.4 on smithi134 2024-07-15T07:03:59.011 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:58 smithi089 bash[82202]: cluster 2024-07-15T07:03:56.864941+0000 mgr.y (mgr.25244) 291 : cluster [DBG] pgmap v233: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:03:59.012 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:58 smithi089 bash[82202]: cephadm 2024-07-15T07:03:57.693741+0000 mgr.y (mgr.25244) 292 : cephadm [INF] Upgrade: Updating osd.4 2024-07-15T07:03:59.012 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:58 smithi089 bash[82202]: audit 2024-07-15T07:03:57.703522+0000 mon.a (mon.0) 516 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:03:59.012 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:58 smithi089 bash[82202]: audit 2024-07-15T07:03:57.705564+0000 mon.a (mon.0) 517 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-07-15T07:03:59.012 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:58 smithi089 bash[82202]: audit 2024-07-15T07:03:57.707047+0000 mon.a (mon.0) 518 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T07:03:59.012 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:03:58 smithi089 bash[82202]: cephadm 2024-07-15T07:03:57.711969+0000 mgr.y (mgr.25244) 293 : cephadm [INF] Deploying daemon osd.4 on smithi134 2024-07-15T07:03:59.013 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:58 smithi134 bash[67624]: cluster 2024-07-15T07:03:56.864941+0000 mgr.y (mgr.25244) 291 : cluster [DBG] pgmap v233: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:03:59.013 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:58 smithi134 bash[67624]: cephadm 2024-07-15T07:03:57.693741+0000 mgr.y (mgr.25244) 292 : cephadm [INF] Upgrade: Updating osd.4 2024-07-15T07:03:59.014 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:58 smithi134 bash[67624]: audit 2024-07-15T07:03:57.703522+0000 mon.a (mon.0) 516 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:03:59.014 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:58 smithi134 bash[67624]: audit 2024-07-15T07:03:57.705564+0000 mon.a (mon.0) 517 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-07-15T07:03:59.014 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:58 smithi134 bash[67624]: audit 2024-07-15T07:03:57.707047+0000 mon.a (mon.0) 518 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T07:03:59.014 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:03:58 smithi134 bash[67624]: cephadm 2024-07-15T07:03:57.711969+0000 mgr.y (mgr.25244) 293 : cephadm [INF] Deploying daemon osd.4 on smithi134 2024-07-15T07:03:59.363 INFO:teuthology.orchestra.run.smithi089.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:badc66b92d2a2cfbbce7f293b5accdc50f486d2b2d966876e46fe28d05f464f1 2024-07-15T07:04:00.197 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -e sha1=6ea7aaa9f93d34dc4d21862afe00140dfddbd706 -- bash -c 'ceph orch ps' 2024-07-15T07:04:00.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:00 smithi089 bash[79810]: cluster 2024-07-15T07:03:58.865701+0000 mgr.y (mgr.25244) 294 : cluster [DBG] pgmap v234: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:04:00.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:00 smithi089 bash[79810]: audit 2024-07-15T07:03:59.359367+0000 mgr.y (mgr.25244) 295 : audit [DBG] from='client.44288 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T07:04:00.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:00 smithi089 bash[79810]: cephadm 2024-07-15T07:03:59.360031+0000 mgr.y (mgr.25244) 296 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:badc66b92d2a2cfbbce7f293b5accdc50f486d2b2d966876e46fe28d05f464f1 2024-07-15T07:04:00.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:00 smithi089 bash[79810]: audit 2024-07-15T07:03:59.366688+0000 mon.a (mon.0) 519 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:04:00.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:00 smithi089 bash[82202]: cluster 2024-07-15T07:03:58.865701+0000 mgr.y (mgr.25244) 294 : cluster [DBG] pgmap v234: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:04:00.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:00 smithi089 bash[82202]: audit 2024-07-15T07:03:59.359367+0000 mgr.y (mgr.25244) 295 : audit [DBG] from='client.44288 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T07:04:00.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:00 smithi089 bash[82202]: cephadm 2024-07-15T07:03:59.360031+0000 mgr.y (mgr.25244) 296 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:badc66b92d2a2cfbbce7f293b5accdc50f486d2b2d966876e46fe28d05f464f1 2024-07-15T07:04:00.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:00 smithi089 bash[82202]: audit 2024-07-15T07:03:59.366688+0000 mon.a (mon.0) 519 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:04:00.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:00 smithi134 bash[67624]: cluster 2024-07-15T07:03:58.865701+0000 mgr.y (mgr.25244) 294 : cluster [DBG] pgmap v234: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:04:00.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:00 smithi134 bash[67624]: audit 2024-07-15T07:03:59.359367+0000 mgr.y (mgr.25244) 295 : audit [DBG] from='client.44288 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T07:04:00.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:00 smithi134 bash[67624]: cephadm 2024-07-15T07:03:59.360031+0000 mgr.y (mgr.25244) 296 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:badc66b92d2a2cfbbce7f293b5accdc50f486d2b2d966876e46fe28d05f464f1 2024-07-15T07:04:00.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:00 smithi134 bash[67624]: audit 2024-07-15T07:03:59.366688+0000 mon.a (mon.0) 519 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:04:02.275 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 07:04:02 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:04:02.275 INFO:journalctl@ceph.osd.5.smithi134.stdout:Jul 15 07:04:02 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:04:02.275 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 07:04:02 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:04:02.276 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:02 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:04:02.276 INFO:journalctl@ceph.osd.4.smithi134.stdout:Jul 15 07:04:02 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:04:02.276 INFO:journalctl@ceph.osd.6.smithi134.stdout:Jul 15 07:04:02 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:04:02.276 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:04:02 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:04:02.277 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:04:02 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:04:02.277 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 07:04:02 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:04:02.555 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:02 smithi134 bash[67624]: cluster 2024-07-15T07:04:00.866613+0000 mgr.y (mgr.25244) 297 : cluster [DBG] pgmap v235: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:04:02.555 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:04:02 smithi134 bash[60729]: ts=2024-07-15T07:04:02.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T07:04:02.555 INFO:journalctl@ceph.osd.4.smithi134.stdout:Jul 15 07:04:02 smithi134 systemd[1]: Stopping Ceph osd.4 for 75ba02bc-4274-11ef-bcad-c7b262605968... 2024-07-15T07:04:02.555 INFO:journalctl@ceph.osd.4.smithi134.stdout:Jul 15 07:04:02 smithi134 bash[27548]: debug 2024-07-15T07:04:02.390+0000 7f61480a2640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-07-15T07:04:02.555 INFO:journalctl@ceph.osd.4.smithi134.stdout:Jul 15 07:04:02 smithi134 bash[27548]: debug 2024-07-15T07:04:02.390+0000 7f61480a2640 -1 osd.4 103 *** Got signal Terminated *** 2024-07-15T07:04:02.555 INFO:journalctl@ceph.osd.4.smithi134.stdout:Jul 15 07:04:02 smithi134 bash[27548]: debug 2024-07-15T07:04:02.390+0000 7f61480a2640 -1 osd.4 103 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-07-15T07:04:02.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:02 smithi089 bash[79810]: cluster 2024-07-15T07:04:00.866613+0000 mgr.y (mgr.25244) 297 : cluster [DBG] pgmap v235: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:04:02.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:02 smithi089 bash[82202]: cluster 2024-07-15T07:04:00.866613+0000 mgr.y (mgr.25244) 297 : cluster [DBG] pgmap v235: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:04:03.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:03 smithi089 bash[82202]: cluster 2024-07-15T07:04:02.394960+0000 mon.a (mon.0) 520 : cluster [INF] osd.4 marked itself down and dead 2024-07-15T07:04:03.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:03 smithi089 bash[79810]: debug 2024-07-15T07:04:03.364+0000 7fb563a0a640 -1 log_channel(cluster) log [ERR] : Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-07-15T07:04:03.622 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:03 smithi089 bash[79810]: cluster 2024-07-15T07:04:02.394960+0000 mon.a (mon.0) 520 : cluster [INF] osd.4 marked itself down and dead 2024-07-15T07:04:03.775 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:03 smithi134 bash[67624]: cluster 2024-07-15T07:04:02.394960+0000 mon.a (mon.0) 520 : cluster [INF] osd.4 marked itself down and dead 2024-07-15T07:04:04.621 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 07:04:04 smithi089 bash[72894]: ::ffff:172.21.15.134 - - [15/Jul/2024:07:04:04] "GET /metrics HTTP/1.1" 200 33836 "" "Prometheus/2.43.0" 2024-07-15T07:04:04.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:04 smithi089 bash[82202]: cluster 2024-07-15T07:04:02.867273+0000 mgr.y (mgr.25244) 298 : cluster [DBG] pgmap v236: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:04:04.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:04 smithi089 bash[82202]: cluster 2024-07-15T07:04:03.370398+0000 mon.a (mon.0) 521 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-07-15T07:04:04.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:04 smithi089 bash[82202]: cluster 2024-07-15T07:04:03.370444+0000 mon.a (mon.0) 522 : cluster [ERR] Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-07-15T07:04:04.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:04 smithi089 bash[82202]: cluster 2024-07-15T07:04:03.389523+0000 mon.a (mon.0) 523 : cluster [DBG] osdmap e104: 8 total, 7 up, 8 in 2024-07-15T07:04:04.622 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:04 smithi089 bash[79810]: cluster 2024-07-15T07:04:02.867273+0000 mgr.y (mgr.25244) 298 : cluster [DBG] pgmap v236: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:04:04.622 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:04 smithi089 bash[79810]: cluster 2024-07-15T07:04:03.370398+0000 mon.a (mon.0) 521 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-07-15T07:04:04.622 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:04 smithi089 bash[79810]: cluster 2024-07-15T07:04:03.370444+0000 mon.a (mon.0) 522 : cluster [ERR] Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-07-15T07:04:04.622 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:04 smithi089 bash[79810]: cluster 2024-07-15T07:04:03.389523+0000 mon.a (mon.0) 523 : cluster [DBG] osdmap e104: 8 total, 7 up, 8 in 2024-07-15T07:04:04.728 INFO:journalctl@ceph.osd.4.smithi134.stdout:Jul 15 07:04:04 smithi134 bash[72838]: ceph-75ba02bc-4274-11ef-bcad-c7b262605968-osd-4 2024-07-15T07:04:04.729 INFO:journalctl@ceph.osd.4.smithi134.stdout:Jul 15 07:04:04 smithi134 bash[72901]: Error response from daemon: No such container: ceph-75ba02bc-4274-11ef-bcad-c7b262605968-osd-4 2024-07-15T07:04:04.729 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:04 smithi134 bash[67624]: cluster 2024-07-15T07:04:02.867273+0000 mgr.y (mgr.25244) 298 : cluster [DBG] pgmap v236: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:04:04.729 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:04 smithi134 bash[67624]: cluster 2024-07-15T07:04:03.370398+0000 mon.a (mon.0) 521 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-07-15T07:04:04.729 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:04 smithi134 bash[67624]: cluster 2024-07-15T07:04:03.370444+0000 mon.a (mon.0) 522 : cluster [ERR] Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-07-15T07:04:04.729 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:04 smithi134 bash[67624]: cluster 2024-07-15T07:04:03.389523+0000 mon.a (mon.0) 523 : cluster [DBG] osdmap e104: 8 total, 7 up, 8 in 2024-07-15T07:04:04.988 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T07:04:05.757 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:05 smithi134 bash[67624]: cluster 2024-07-15T07:04:04.391892+0000 mon.a (mon.0) 524 : cluster [DBG] osdmap e105: 8 total, 7 up, 8 in 2024-07-15T07:04:05.757 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:05 smithi134 bash[67624]: audit 2024-07-15T07:04:04.800622+0000 mon.a (mon.0) 525 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:04:05.757 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:05 smithi134 bash[67624]: audit 2024-07-15T07:04:04.802879+0000 mon.a (mon.0) 526 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:04:05.757 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:05 smithi134 bash[67624]: audit 2024-07-15T07:04:04.976436+0000 mon.a (mon.0) 527 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.5", "id": [3, 4]}]: dispatch 2024-07-15T07:04:05.757 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:05 smithi134 bash[67624]: audit 2024-07-15T07:04:04.977086+0000 mon.a (mon.0) 528 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.d", "id": [7, 4]}]: dispatch 2024-07-15T07:04:05.757 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:05 smithi134 bash[67624]: audit 2024-07-15T07:04:04.977712+0000 mon.a (mon.0) 529 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.11", "id": [3, 4]}]: dispatch 2024-07-15T07:04:05.757 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:05 smithi134 bash[67624]: audit 2024-07-15T07:04:04.978313+0000 mon.a (mon.0) 530 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [3, 2, 1, 4]}]: dispatch 2024-07-15T07:04:05.757 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:05 smithi134 bash[67624]: audit 2024-07-15T07:04:04.978911+0000 mon.a (mon.0) 531 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [3, 5]}]: dispatch 2024-07-15T07:04:05.758 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:05 smithi134 bash[67624]: audit 2024-07-15T07:04:04.979499+0000 mon.a (mon.0) 532 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.c"}]: dispatch 2024-07-15T07:04:05.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:05 smithi089 bash[79810]: cluster 2024-07-15T07:04:04.391892+0000 mon.a (mon.0) 524 : cluster [DBG] osdmap e105: 8 total, 7 up, 8 in 2024-07-15T07:04:05.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:05 smithi089 bash[79810]: audit 2024-07-15T07:04:04.800622+0000 mon.a (mon.0) 525 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:04:05.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:05 smithi089 bash[79810]: audit 2024-07-15T07:04:04.802879+0000 mon.a (mon.0) 526 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:04:05.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:05 smithi089 bash[79810]: audit 2024-07-15T07:04:04.976436+0000 mon.a (mon.0) 527 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.5", "id": [3, 4]}]: dispatch 2024-07-15T07:04:05.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:05 smithi089 bash[79810]: audit 2024-07-15T07:04:04.977086+0000 mon.a (mon.0) 528 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.d", "id": [7, 4]}]: dispatch 2024-07-15T07:04:05.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:05 smithi089 bash[79810]: audit 2024-07-15T07:04:04.977712+0000 mon.a (mon.0) 529 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.11", "id": [3, 4]}]: dispatch 2024-07-15T07:04:05.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:05 smithi089 bash[79810]: audit 2024-07-15T07:04:04.978313+0000 mon.a (mon.0) 530 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [3, 2, 1, 4]}]: dispatch 2024-07-15T07:04:05.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:05 smithi089 bash[79810]: audit 2024-07-15T07:04:04.978911+0000 mon.a (mon.0) 531 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [3, 5]}]: dispatch 2024-07-15T07:04:05.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:05 smithi089 bash[79810]: audit 2024-07-15T07:04:04.979499+0000 mon.a (mon.0) 532 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.c"}]: dispatch 2024-07-15T07:04:05.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:05 smithi089 bash[82202]: cluster 2024-07-15T07:04:04.391892+0000 mon.a (mon.0) 524 : cluster [DBG] osdmap e105: 8 total, 7 up, 8 in 2024-07-15T07:04:05.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:05 smithi089 bash[82202]: audit 2024-07-15T07:04:04.800622+0000 mon.a (mon.0) 525 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:04:05.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:05 smithi089 bash[82202]: audit 2024-07-15T07:04:04.802879+0000 mon.a (mon.0) 526 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:04:05.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:05 smithi089 bash[82202]: audit 2024-07-15T07:04:04.976436+0000 mon.a (mon.0) 527 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.5", "id": [3, 4]}]: dispatch 2024-07-15T07:04:05.873 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:05 smithi089 bash[82202]: audit 2024-07-15T07:04:04.977086+0000 mon.a (mon.0) 528 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.d", "id": [7, 4]}]: dispatch 2024-07-15T07:04:05.873 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:05 smithi089 bash[82202]: audit 2024-07-15T07:04:04.977712+0000 mon.a (mon.0) 529 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.11", "id": [3, 4]}]: dispatch 2024-07-15T07:04:05.873 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:05 smithi089 bash[82202]: audit 2024-07-15T07:04:04.978313+0000 mon.a (mon.0) 530 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [3, 2, 1, 4]}]: dispatch 2024-07-15T07:04:05.873 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:05 smithi089 bash[82202]: audit 2024-07-15T07:04:04.978911+0000 mon.a (mon.0) 531 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [3, 5]}]: dispatch 2024-07-15T07:04:05.873 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:05 smithi089 bash[82202]: audit 2024-07-15T07:04:04.979499+0000 mon.a (mon.0) 532 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.c"}]: dispatch 2024-07-15T07:04:06.982 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 07:04:06 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:04:06.983 INFO:journalctl@ceph.osd.5.smithi134.stdout:Jul 15 07:04:06 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:04:06.983 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:04:06 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:04:06.983 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:04:06 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:04:06.984 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 07:04:06 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:04:06.984 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 07:04:06 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:04:06.984 INFO:journalctl@ceph.osd.4.smithi134.stdout:Jul 15 07:04:06 smithi134 systemd[1]: ceph-75ba02bc-4274-11ef-bcad-c7b262605968@osd.4.service: Deactivated successfully. 2024-07-15T07:04:06.984 INFO:journalctl@ceph.osd.4.smithi134.stdout:Jul 15 07:04:06 smithi134 systemd[1]: Stopped Ceph osd.4 for 75ba02bc-4274-11ef-bcad-c7b262605968. 2024-07-15T07:04:06.984 INFO:journalctl@ceph.osd.4.smithi134.stdout:Jul 15 07:04:06 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:04:06.985 INFO:journalctl@ceph.osd.6.smithi134.stdout:Jul 15 07:04:06 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:04:06.985 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:06 smithi134 bash[67624]: cluster 2024-07-15T07:04:04.867857+0000 mgr.y (mgr.25244) 299 : cluster [DBG] pgmap v239: 73 pgs: 11 stale+active+clean, 62 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:04:06.985 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:06 smithi134 bash[67624]: audit 2024-07-15T07:04:05.809366+0000 mon.a (mon.0) 533 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.5", "id": [3, 4]}]': finished 2024-07-15T07:04:06.985 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:06 smithi134 bash[67624]: audit 2024-07-15T07:04:05.809588+0000 mon.a (mon.0) 534 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.d", "id": [7, 4]}]': finished 2024-07-15T07:04:06.985 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:06 smithi134 bash[67624]: audit 2024-07-15T07:04:05.809810+0000 mon.a (mon.0) 535 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.11", "id": [3, 4]}]': finished 2024-07-15T07:04:06.985 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:06 smithi134 bash[67624]: audit 2024-07-15T07:04:05.810072+0000 mon.a (mon.0) 536 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [3, 2, 1, 4]}]': finished 2024-07-15T07:04:06.985 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:06 smithi134 bash[67624]: audit 2024-07-15T07:04:05.810288+0000 mon.a (mon.0) 537 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [3, 5]}]': finished 2024-07-15T07:04:06.985 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:06 smithi134 bash[67624]: audit 2024-07-15T07:04:05.810531+0000 mon.a (mon.0) 538 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd='[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.c"}]': finished 2024-07-15T07:04:06.985 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:06 smithi134 bash[67624]: cluster 2024-07-15T07:04:05.810690+0000 mon.a (mon.0) 539 : cluster [DBG] osdmap e106: 8 total, 7 up, 8 in 2024-07-15T07:04:06.985 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:06 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:04:07.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:06 smithi089 bash[82202]: cluster 2024-07-15T07:04:04.867857+0000 mgr.y (mgr.25244) 299 : cluster [DBG] pgmap v239: 73 pgs: 11 stale+active+clean, 62 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:04:07.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:06 smithi089 bash[82202]: audit 2024-07-15T07:04:05.809366+0000 mon.a (mon.0) 533 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.5", "id": [3, 4]}]': finished 2024-07-15T07:04:07.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:06 smithi089 bash[82202]: audit 2024-07-15T07:04:05.809588+0000 mon.a (mon.0) 534 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.d", "id": [7, 4]}]': finished 2024-07-15T07:04:07.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:06 smithi089 bash[82202]: audit 2024-07-15T07:04:05.809810+0000 mon.a (mon.0) 535 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.11", "id": [3, 4]}]': finished 2024-07-15T07:04:07.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:06 smithi089 bash[82202]: audit 2024-07-15T07:04:05.810072+0000 mon.a (mon.0) 536 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [3, 2, 1, 4]}]': finished 2024-07-15T07:04:07.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:06 smithi089 bash[82202]: audit 2024-07-15T07:04:05.810288+0000 mon.a (mon.0) 537 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [3, 5]}]': finished 2024-07-15T07:04:07.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:06 smithi089 bash[82202]: audit 2024-07-15T07:04:05.810531+0000 mon.a (mon.0) 538 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd='[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.c"}]': finished 2024-07-15T07:04:07.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:06 smithi089 bash[82202]: cluster 2024-07-15T07:04:05.810690+0000 mon.a (mon.0) 539 : cluster [DBG] osdmap e106: 8 total, 7 up, 8 in 2024-07-15T07:04:07.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:06 smithi089 bash[79810]: cluster 2024-07-15T07:04:04.867857+0000 mgr.y (mgr.25244) 299 : cluster [DBG] pgmap v239: 73 pgs: 11 stale+active+clean, 62 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:04:07.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:06 smithi089 bash[79810]: audit 2024-07-15T07:04:05.809366+0000 mon.a (mon.0) 533 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.5", "id": [3, 4]}]': finished 2024-07-15T07:04:07.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:06 smithi089 bash[79810]: audit 2024-07-15T07:04:05.809588+0000 mon.a (mon.0) 534 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.d", "id": [7, 4]}]': finished 2024-07-15T07:04:07.123 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:06 smithi089 bash[79810]: audit 2024-07-15T07:04:05.809810+0000 mon.a (mon.0) 535 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.11", "id": [3, 4]}]': finished 2024-07-15T07:04:07.123 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:06 smithi089 bash[79810]: audit 2024-07-15T07:04:05.810072+0000 mon.a (mon.0) 536 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [3, 2, 1, 4]}]': finished 2024-07-15T07:04:07.123 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:06 smithi089 bash[79810]: audit 2024-07-15T07:04:05.810288+0000 mon.a (mon.0) 537 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [3, 5]}]': finished 2024-07-15T07:04:07.123 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:06 smithi089 bash[79810]: audit 2024-07-15T07:04:05.810531+0000 mon.a (mon.0) 538 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd='[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.c"}]': finished 2024-07-15T07:04:07.123 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:06 smithi089 bash[79810]: cluster 2024-07-15T07:04:05.810690+0000 mon.a (mon.0) 539 : cluster [DBG] osdmap e106: 8 total, 7 up, 8 in 2024-07-15T07:04:07.304 INFO:journalctl@ceph.osd.4.smithi134.stdout:Jul 15 07:04:06 smithi134 systemd[1]: Started Ceph osd.4 for 75ba02bc-4274-11ef-bcad-c7b262605968. 2024-07-15T07:04:08.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:07 smithi089 bash[79810]: cluster 2024-07-15T07:04:06.814706+0000 mon.a (mon.0) 540 : cluster [DBG] osdmap e107: 8 total, 7 up, 8 in 2024-07-15T07:04:08.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:07 smithi089 bash[79810]: audit 2024-07-15T07:04:07.034843+0000 mon.a (mon.0) 541 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:04:08.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:07 smithi089 bash[79810]: audit 2024-07-15T07:04:07.045220+0000 mon.a (mon.0) 542 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:04:08.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:07 smithi089 bash[79810]: audit 2024-07-15T07:04:07.047427+0000 mon.a (mon.0) 543 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T07:04:08.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:07 smithi089 bash[79810]: cluster 2024-07-15T07:04:07.816381+0000 mon.a (mon.0) 544 : cluster [DBG] osdmap e108: 8 total, 7 up, 8 in 2024-07-15T07:04:08.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:07 smithi089 bash[82202]: cluster 2024-07-15T07:04:06.814706+0000 mon.a (mon.0) 540 : cluster [DBG] osdmap e107: 8 total, 7 up, 8 in 2024-07-15T07:04:08.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:07 smithi089 bash[82202]: audit 2024-07-15T07:04:07.034843+0000 mon.a (mon.0) 541 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:04:08.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:07 smithi089 bash[82202]: audit 2024-07-15T07:04:07.045220+0000 mon.a (mon.0) 542 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:04:08.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:07 smithi089 bash[82202]: audit 2024-07-15T07:04:07.047427+0000 mon.a (mon.0) 543 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T07:04:08.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:07 smithi089 bash[82202]: cluster 2024-07-15T07:04:07.816381+0000 mon.a (mon.0) 544 : cluster [DBG] osdmap e108: 8 total, 7 up, 8 in 2024-07-15T07:04:08.138 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:07 smithi134 bash[67624]: cluster 2024-07-15T07:04:06.814706+0000 mon.a (mon.0) 540 : cluster [DBG] osdmap e107: 8 total, 7 up, 8 in 2024-07-15T07:04:08.138 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:07 smithi134 bash[67624]: audit 2024-07-15T07:04:07.034843+0000 mon.a (mon.0) 541 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:04:08.138 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:07 smithi134 bash[67624]: audit 2024-07-15T07:04:07.045220+0000 mon.a (mon.0) 542 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:04:08.138 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:07 smithi134 bash[67624]: audit 2024-07-15T07:04:07.047427+0000 mon.a (mon.0) 543 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T07:04:08.138 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:07 smithi134 bash[67624]: cluster 2024-07-15T07:04:07.816381+0000 mon.a (mon.0) 544 : cluster [DBG] osdmap e108: 8 total, 7 up, 8 in 2024-07-15T07:04:08.805 INFO:journalctl@ceph.osd.4.smithi134.stdout:Jul 15 07:04:08 smithi134 bash[73048]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-07-15T07:04:08.805 INFO:journalctl@ceph.osd.4.smithi134.stdout:Jul 15 07:04:08 smithi134 bash[73048]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-07-15T07:04:08.805 INFO:journalctl@ceph.osd.4.smithi134.stdout:Jul 15 07:04:08 smithi134 bash[73048]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-07-15T07:04:08.805 INFO:journalctl@ceph.osd.4.smithi134.stdout:Jul 15 07:04:08 smithi134 bash[73048]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-07-15T07:04:08.805 INFO:journalctl@ceph.osd.4.smithi134.stdout:Jul 15 07:04:08 smithi134 bash[73048]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-07-15T07:04:08.805 INFO:journalctl@ceph.osd.4.smithi134.stdout:Jul 15 07:04:08 smithi134 bash[73048]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-07-15T07:04:08.805 INFO:journalctl@ceph.osd.4.smithi134.stdout:Jul 15 07:04:08 smithi134 bash[73048]: --> ceph-volume raw activate successful for osd ID: 4 2024-07-15T07:04:09.119 INFO:teuthology.orchestra.run.smithi089.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-07-15T07:04:09.119 INFO:teuthology.orchestra.run.smithi089.stdout:alertmanager.a smithi089 *:9093,9094 running (8m) 25s ago 16m 14.1M - 0.25.0 c8568f914cd2 c476af8941dd 2024-07-15T07:04:09.120 INFO:teuthology.orchestra.run.smithi089.stdout:grafana.a smithi134 *:3000 running (8m) 55s ago 15m 75.0M - 9.4.7 954c08fa6188 ac957811e4af 2024-07-15T07:04:09.120 INFO:teuthology.orchestra.run.smithi089.stdout:mds.foo.smithi089.cczuoz smithi089 running (13m) 25s ago 13m 14.8M - 17.2.7-1102-g9a202024 280b03ed6869 a1f028849336 2024-07-15T07:04:09.120 INFO:teuthology.orchestra.run.smithi089.stdout:mds.foo.smithi134.wugtzh smithi134 running (13m) 55s ago 13m 16.9M - 17.2.7-1102-g9a202024 280b03ed6869 fb5bbeafa32b 2024-07-15T07:04:09.120 INFO:teuthology.orchestra.run.smithi089.stdout:mgr.x smithi134 *:8443,9283,8765 running (7m) 55s ago 24m 425M - 18.2.2-1609-g6ea7aaa9 0add76435ed9 842a5eaedc06 2024-07-15T07:04:09.120 INFO:teuthology.orchestra.run.smithi089.stdout:mgr.y smithi089 *:8443,9283,8765 running (8m) 25s ago 26m 482M - 18.2.2-1609-g6ea7aaa9 0add76435ed9 20c56e260e99 2024-07-15T07:04:09.120 INFO:teuthology.orchestra.run.smithi089.stdout:mon.a smithi089 running (7m) 25s ago 27m 48.0M 2048M 18.2.2-1609-g6ea7aaa9 0add76435ed9 b2d2728b9d78 2024-07-15T07:04:09.120 INFO:teuthology.orchestra.run.smithi089.stdout:mon.b smithi134 running (4m) 55s ago 24m 36.5M 2048M 18.2.2-1609-g6ea7aaa9 0add76435ed9 f157e925047f 2024-07-15T07:04:09.120 INFO:teuthology.orchestra.run.smithi089.stdout:mon.c smithi089 running (5m) 25s ago 24m 41.9M 2048M 18.2.2-1609-g6ea7aaa9 0add76435ed9 60c08b0bcea8 2024-07-15T07:04:09.120 INFO:teuthology.orchestra.run.smithi089.stdout:node-exporter.a smithi089 *:9100 running (8m) 25s ago 16m 8511k - 1.5.0 0da6a335fe13 5da5d79c182f 2024-07-15T07:04:09.120 INFO:teuthology.orchestra.run.smithi089.stdout:node-exporter.b smithi134 *:9100 running (8m) 55s ago 16m 8324k - 1.5.0 0da6a335fe13 9ace5df7006d 2024-07-15T07:04:09.120 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0 smithi089 running (3m) 25s ago 23m 49.2M 2524M 18.2.2-1609-g6ea7aaa9 0add76435ed9 62162797039a 2024-07-15T07:04:09.120 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1 smithi089 running (2m) 25s ago 22m 42.2M 2524M 18.2.2-1609-g6ea7aaa9 0add76435ed9 fa4122a785bf 2024-07-15T07:04:09.121 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2 smithi089 running (59s) 25s ago 21m 48.7M 2524M 18.2.2-1609-g6ea7aaa9 0add76435ed9 b09291d6c7bd 2024-07-15T07:04:09.121 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3 smithi089 running (29s) 25s ago 21m - 2524M 18.2.2-1609-g6ea7aaa9 0add76435ed9 92c510e458f9 2024-07-15T07:04:09.121 INFO:teuthology.orchestra.run.smithi089.stdout:osd.4 smithi134 starting - - - 2524M 2024-07-15T07:04:09.121 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5 smithi134 running (19m) 55s ago 19m 55.7M 2524M 17.2.7-1102-g9a202024 280b03ed6869 61be14c58fc3 2024-07-15T07:04:09.121 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6 smithi134 running (18m) 55s ago 18m 55.1M 2524M 17.2.7-1102-g9a202024 280b03ed6869 f92c7dd84666 2024-07-15T07:04:09.121 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7 smithi134 running (17m) 55s ago 17m 55.7M 2524M 17.2.7-1102-g9a202024 280b03ed6869 e44d11319ff6 2024-07-15T07:04:09.121 INFO:teuthology.orchestra.run.smithi089.stdout:prometheus.a smithi134 *:9095 running (7m) 55s ago 16m 54.5M - 2.43.0 a07b618ecd1d 8873e527bacf 2024-07-15T07:04:09.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:08 smithi089 bash[79810]: cluster 2024-07-15T07:04:06.868624+0000 mgr.y (mgr.25244) 300 : cluster [DBG] pgmap v242: 73 pgs: 18 active+undersized, 1 stale+active+clean, 8 active+undersized+degraded, 46 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 13/72 objects degraded (18.056%) 2024-07-15T07:04:09.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:08 smithi089 bash[79810]: cluster 2024-07-15T07:04:07.820669+0000 mon.a (mon.0) 545 : cluster [WRN] Health check failed: Degraded data redundancy: 13/72 objects degraded (18.056%), 8 pgs degraded (PG_DEGRADED) 2024-07-15T07:04:09.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:08 smithi089 bash[82202]: cluster 2024-07-15T07:04:06.868624+0000 mgr.y (mgr.25244) 300 : cluster [DBG] pgmap v242: 73 pgs: 18 active+undersized, 1 stale+active+clean, 8 active+undersized+degraded, 46 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 13/72 objects degraded (18.056%) 2024-07-15T07:04:09.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:08 smithi089 bash[82202]: cluster 2024-07-15T07:04:07.820669+0000 mon.a (mon.0) 545 : cluster [WRN] Health check failed: Degraded data redundancy: 13/72 objects degraded (18.056%), 8 pgs degraded (PG_DEGRADED) 2024-07-15T07:04:09.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:08 smithi134 bash[67624]: cluster 2024-07-15T07:04:06.868624+0000 mgr.y (mgr.25244) 300 : cluster [DBG] pgmap v242: 73 pgs: 18 active+undersized, 1 stale+active+clean, 8 active+undersized+degraded, 46 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 13/72 objects degraded (18.056%) 2024-07-15T07:04:09.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:08 smithi134 bash[67624]: cluster 2024-07-15T07:04:07.820669+0000 mon.a (mon.0) 545 : cluster [WRN] Health check failed: Degraded data redundancy: 13/72 objects degraded (18.056%), 8 pgs degraded (PG_DEGRADED) 2024-07-15T07:04:09.901 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -e sha1=6ea7aaa9f93d34dc4d21862afe00140dfddbd706 -- bash -c 'ceph orch ps' 2024-07-15T07:04:11.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:10 smithi089 bash[79810]: cluster 2024-07-15T07:04:08.869150+0000 mgr.y (mgr.25244) 301 : cluster [DBG] pgmap v244: 73 pgs: 22 active+undersized, 9 active+undersized+degraded, 42 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 14/72 objects degraded (19.444%) 2024-07-15T07:04:11.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:10 smithi089 bash[79810]: audit 2024-07-15T07:04:09.087657+0000 mgr.y (mgr.25244) 302 : audit [DBG] from='client.44294 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T07:04:11.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:10 smithi089 bash[82202]: cluster 2024-07-15T07:04:08.869150+0000 mgr.y (mgr.25244) 301 : cluster [DBG] pgmap v244: 73 pgs: 22 active+undersized, 9 active+undersized+degraded, 42 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 14/72 objects degraded (19.444%) 2024-07-15T07:04:11.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:10 smithi089 bash[82202]: audit 2024-07-15T07:04:09.087657+0000 mgr.y (mgr.25244) 302 : audit [DBG] from='client.44294 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T07:04:11.305 INFO:journalctl@ceph.osd.4.smithi134.stdout:Jul 15 07:04:10 smithi134 bash[73271]: debug 2024-07-15T07:04:10.914+0000 7fcf6cb85640 -1 Falling back to public interface 2024-07-15T07:04:11.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:10 smithi134 bash[67624]: cluster 2024-07-15T07:04:08.869150+0000 mgr.y (mgr.25244) 301 : cluster [DBG] pgmap v244: 73 pgs: 22 active+undersized, 9 active+undersized+degraded, 42 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 14/72 objects degraded (19.444%) 2024-07-15T07:04:11.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:10 smithi134 bash[67624]: audit 2024-07-15T07:04:09.087657+0000 mgr.y (mgr.25244) 302 : audit [DBG] from='client.44294 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T07:04:12.805 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:04:12 smithi134 bash[60729]: ts=2024-07-15T07:04:12.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T07:04:13.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:12 smithi089 bash[82202]: cluster 2024-07-15T07:04:10.870276+0000 mgr.y (mgr.25244) 303 : cluster [DBG] pgmap v245: 73 pgs: 6 active+clean+remapped, 22 active+undersized, 9 active+undersized+degraded, 36 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 14/72 objects degraded (19.444%); 1/72 objects misplaced (1.389%) 2024-07-15T07:04:13.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:12 smithi089 bash[79810]: cluster 2024-07-15T07:04:10.870276+0000 mgr.y (mgr.25244) 303 : cluster [DBG] pgmap v245: 73 pgs: 6 active+clean+remapped, 22 active+undersized, 9 active+undersized+degraded, 36 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 14/72 objects degraded (19.444%); 1/72 objects misplaced (1.389%) 2024-07-15T07:04:13.304 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:12 smithi134 bash[67624]: cluster 2024-07-15T07:04:10.870276+0000 mgr.y (mgr.25244) 303 : cluster [DBG] pgmap v245: 73 pgs: 6 active+clean+remapped, 22 active+undersized, 9 active+undersized+degraded, 36 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 14/72 objects degraded (19.444%); 1/72 objects misplaced (1.389%) 2024-07-15T07:04:14.305 INFO:journalctl@ceph.osd.4.smithi134.stdout:Jul 15 07:04:13 smithi134 bash[73271]: debug 2024-07-15T07:04:13.930+0000 7fcf6cb85640 -1 osd.4 103 log_to_monitors true 2024-07-15T07:04:14.621 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 07:04:14 smithi089 bash[72894]: ::ffff:172.21.15.134 - - [15/Jul/2024:07:04:14] "GET /metrics HTTP/1.1" 200 33983 "" "Prometheus/2.43.0" 2024-07-15T07:04:14.685 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T07:04:15.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:14 smithi089 bash[79810]: cluster 2024-07-15T07:04:12.871148+0000 mgr.y (mgr.25244) 304 : cluster [DBG] pgmap v246: 73 pgs: 6 active+clean+remapped, 22 active+undersized, 9 active+undersized+degraded, 36 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 14/72 objects degraded (19.444%); 1/72 objects misplaced (1.389%) 2024-07-15T07:04:15.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:14 smithi089 bash[79810]: audit 2024-07-15T07:04:13.860085+0000 mon.a (mon.0) 546 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:04:15.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:14 smithi089 bash[79810]: audit 2024-07-15T07:04:13.872164+0000 mon.a (mon.0) 547 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:04:15.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:14 smithi089 bash[79810]: audit 2024-07-15T07:04:13.939079+0000 mon.a (mon.0) 548 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-07-15T07:04:15.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:14 smithi089 bash[79810]: audit 2024-07-15T07:04:13.939613+0000 mon.b (mon.1) 4 : audit [INF] from='osd.4 [v2:172.21.15.134:6800/1882540973,v1:172.21.15.134:6801/1882540973]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-07-15T07:04:15.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:14 smithi089 bash[79810]: audit 2024-07-15T07:04:14.662630+0000 mon.a (mon.0) 549 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:04:15.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:14 smithi089 bash[79810]: audit 2024-07-15T07:04:14.672745+0000 mon.a (mon.0) 550 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:04:15.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:14 smithi089 bash[82202]: cluster 2024-07-15T07:04:12.871148+0000 mgr.y (mgr.25244) 304 : cluster [DBG] pgmap v246: 73 pgs: 6 active+clean+remapped, 22 active+undersized, 9 active+undersized+degraded, 36 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 14/72 objects degraded (19.444%); 1/72 objects misplaced (1.389%) 2024-07-15T07:04:15.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:14 smithi089 bash[82202]: audit 2024-07-15T07:04:13.860085+0000 mon.a (mon.0) 546 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:04:15.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:14 smithi089 bash[82202]: audit 2024-07-15T07:04:13.872164+0000 mon.a (mon.0) 547 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:04:15.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:14 smithi089 bash[82202]: audit 2024-07-15T07:04:13.939079+0000 mon.a (mon.0) 548 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-07-15T07:04:15.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:14 smithi089 bash[82202]: audit 2024-07-15T07:04:13.939613+0000 mon.b (mon.1) 4 : audit [INF] from='osd.4 [v2:172.21.15.134:6800/1882540973,v1:172.21.15.134:6801/1882540973]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-07-15T07:04:15.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:14 smithi089 bash[82202]: audit 2024-07-15T07:04:14.662630+0000 mon.a (mon.0) 549 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:04:15.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:14 smithi089 bash[82202]: audit 2024-07-15T07:04:14.672745+0000 mon.a (mon.0) 550 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:04:15.305 INFO:journalctl@ceph.osd.4.smithi134.stdout:Jul 15 07:04:14 smithi134 bash[73271]: debug 2024-07-15T07:04:14.914+0000 7fcf6492f640 -1 osd.4 103 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-07-15T07:04:15.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:14 smithi134 bash[67624]: cluster 2024-07-15T07:04:12.871148+0000 mgr.y (mgr.25244) 304 : cluster [DBG] pgmap v246: 73 pgs: 6 active+clean+remapped, 22 active+undersized, 9 active+undersized+degraded, 36 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 14/72 objects degraded (19.444%); 1/72 objects misplaced (1.389%) 2024-07-15T07:04:15.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:14 smithi134 bash[67624]: audit 2024-07-15T07:04:13.860085+0000 mon.a (mon.0) 546 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:04:15.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:14 smithi134 bash[67624]: audit 2024-07-15T07:04:13.872164+0000 mon.a (mon.0) 547 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:04:15.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:14 smithi134 bash[67624]: audit 2024-07-15T07:04:13.939079+0000 mon.a (mon.0) 548 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-07-15T07:04:15.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:14 smithi134 bash[67624]: audit 2024-07-15T07:04:13.939613+0000 mon.b (mon.1) 4 : audit [INF] from='osd.4 [v2:172.21.15.134:6800/1882540973,v1:172.21.15.134:6801/1882540973]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-07-15T07:04:15.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:14 smithi134 bash[67624]: audit 2024-07-15T07:04:14.662630+0000 mon.a (mon.0) 549 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:04:15.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:14 smithi134 bash[67624]: audit 2024-07-15T07:04:14.672745+0000 mon.a (mon.0) 550 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:04:16.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:15 smithi089 bash[79810]: debug 2024-07-15T07:04:15.864+0000 7fb563a0a640 -1 log_channel(cluster) log [ERR] : Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-07-15T07:04:16.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:15 smithi089 bash[79810]: cluster 2024-07-15T07:04:14.872047+0000 mgr.y (mgr.25244) 305 : cluster [DBG] pgmap v248: 73 pgs: 6 active+clean+remapped, 22 active+undersized, 9 active+undersized+degraded, 36 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 14/72 objects degraded (19.444%); 1/72 objects misplaced (1.389%) 2024-07-15T07:04:16.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:15 smithi089 bash[79810]: audit 2024-07-15T07:04:14.873954+0000 mon.a (mon.0) 551 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-07-15T07:04:16.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:15 smithi089 bash[79810]: cluster 2024-07-15T07:04:14.874090+0000 mon.a (mon.0) 552 : cluster [DBG] osdmap e109: 8 total, 7 up, 8 in 2024-07-15T07:04:16.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:15 smithi089 bash[79810]: audit 2024-07-15T07:04:14.877205+0000 mon.a (mon.0) 553 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]: dispatch 2024-07-15T07:04:16.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:15 smithi089 bash[79810]: audit 2024-07-15T07:04:14.877867+0000 mon.b (mon.1) 5 : audit [INF] from='osd.4 [v2:172.21.15.134:6800/1882540973,v1:172.21.15.134:6801/1882540973]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]: dispatch 2024-07-15T07:04:16.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:15 smithi089 bash[82202]: cluster 2024-07-15T07:04:14.872047+0000 mgr.y (mgr.25244) 305 : cluster [DBG] pgmap v248: 73 pgs: 6 active+clean+remapped, 22 active+undersized, 9 active+undersized+degraded, 36 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 14/72 objects degraded (19.444%); 1/72 objects misplaced (1.389%) 2024-07-15T07:04:16.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:15 smithi089 bash[82202]: audit 2024-07-15T07:04:14.873954+0000 mon.a (mon.0) 551 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-07-15T07:04:16.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:15 smithi089 bash[82202]: cluster 2024-07-15T07:04:14.874090+0000 mon.a (mon.0) 552 : cluster [DBG] osdmap e109: 8 total, 7 up, 8 in 2024-07-15T07:04:16.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:15 smithi089 bash[82202]: audit 2024-07-15T07:04:14.877205+0000 mon.a (mon.0) 553 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]: dispatch 2024-07-15T07:04:16.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:15 smithi089 bash[82202]: audit 2024-07-15T07:04:14.877867+0000 mon.b (mon.1) 5 : audit [INF] from='osd.4 [v2:172.21.15.134:6800/1882540973,v1:172.21.15.134:6801/1882540973]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]: dispatch 2024-07-15T07:04:16.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:15 smithi134 bash[67624]: cluster 2024-07-15T07:04:14.872047+0000 mgr.y (mgr.25244) 305 : cluster [DBG] pgmap v248: 73 pgs: 6 active+clean+remapped, 22 active+undersized, 9 active+undersized+degraded, 36 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 14/72 objects degraded (19.444%); 1/72 objects misplaced (1.389%) 2024-07-15T07:04:16.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:15 smithi134 bash[67624]: audit 2024-07-15T07:04:14.873954+0000 mon.a (mon.0) 551 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-07-15T07:04:16.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:15 smithi134 bash[67624]: cluster 2024-07-15T07:04:14.874090+0000 mon.a (mon.0) 552 : cluster [DBG] osdmap e109: 8 total, 7 up, 8 in 2024-07-15T07:04:16.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:15 smithi134 bash[67624]: audit 2024-07-15T07:04:14.877205+0000 mon.a (mon.0) 553 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]: dispatch 2024-07-15T07:04:16.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:15 smithi134 bash[67624]: audit 2024-07-15T07:04:14.877867+0000 mon.b (mon.1) 5 : audit [INF] from='osd.4 [v2:172.21.15.134:6800/1882540973,v1:172.21.15.134:6801/1882540973]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]: dispatch 2024-07-15T07:04:17.170 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:16 smithi089 bash[82202]: cluster 2024-07-15T07:04:15.869624+0000 mon.a (mon.0) 554 : cluster [ERR] Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-07-15T07:04:17.170 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:16 smithi089 bash[82202]: cluster 2024-07-15T07:04:15.869669+0000 mon.a (mon.0) 555 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-07-15T07:04:17.170 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:16 smithi089 bash[82202]: cluster 2024-07-15T07:04:15.879169+0000 mon.a (mon.0) 556 : cluster [INF] osd.4 [v2:172.21.15.134:6800/1882540973,v1:172.21.15.134:6801/1882540973] boot 2024-07-15T07:04:17.170 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:16 smithi089 bash[82202]: cluster 2024-07-15T07:04:15.879301+0000 mon.a (mon.0) 557 : cluster [DBG] osdmap e110: 8 total, 8 up, 8 in 2024-07-15T07:04:17.171 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:16 smithi089 bash[82202]: audit 2024-07-15T07:04:15.883242+0000 mon.a (mon.0) 558 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-07-15T07:04:17.171 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:16 smithi089 bash[79810]: cluster 2024-07-15T07:04:15.869624+0000 mon.a (mon.0) 554 : cluster [ERR] Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-07-15T07:04:17.171 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:16 smithi089 bash[79810]: cluster 2024-07-15T07:04:15.869669+0000 mon.a (mon.0) 555 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-07-15T07:04:17.171 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:16 smithi089 bash[79810]: cluster 2024-07-15T07:04:15.879169+0000 mon.a (mon.0) 556 : cluster [INF] osd.4 [v2:172.21.15.134:6800/1882540973,v1:172.21.15.134:6801/1882540973] boot 2024-07-15T07:04:17.171 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:16 smithi089 bash[79810]: cluster 2024-07-15T07:04:15.879301+0000 mon.a (mon.0) 557 : cluster [DBG] osdmap e110: 8 total, 8 up, 8 in 2024-07-15T07:04:17.171 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:16 smithi089 bash[79810]: audit 2024-07-15T07:04:15.883242+0000 mon.a (mon.0) 558 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-07-15T07:04:17.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:16 smithi134 bash[67624]: cluster 2024-07-15T07:04:15.869624+0000 mon.a (mon.0) 554 : cluster [ERR] Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-07-15T07:04:17.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:16 smithi134 bash[67624]: cluster 2024-07-15T07:04:15.869669+0000 mon.a (mon.0) 555 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-07-15T07:04:17.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:16 smithi134 bash[67624]: cluster 2024-07-15T07:04:15.879169+0000 mon.a (mon.0) 556 : cluster [INF] osd.4 [v2:172.21.15.134:6800/1882540973,v1:172.21.15.134:6801/1882540973] boot 2024-07-15T07:04:17.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:16 smithi134 bash[67624]: cluster 2024-07-15T07:04:15.879301+0000 mon.a (mon.0) 557 : cluster [DBG] osdmap e110: 8 total, 8 up, 8 in 2024-07-15T07:04:17.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:16 smithi134 bash[67624]: audit 2024-07-15T07:04:15.883242+0000 mon.a (mon.0) 558 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-07-15T07:04:18.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:17 smithi089 bash[79810]: cluster 2024-07-15T07:04:16.873009+0000 mgr.y (mgr.25244) 306 : cluster [DBG] pgmap v250: 73 pgs: 3 peering, 6 active+clean+remapped, 19 active+undersized, 9 active+undersized+degraded, 36 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 14/72 objects degraded (19.444%); 1/72 objects misplaced (1.389%) 2024-07-15T07:04:18.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:17 smithi089 bash[79810]: cluster 2024-07-15T07:04:16.888503+0000 mon.a (mon.0) 559 : cluster [DBG] osdmap e111: 8 total, 8 up, 8 in 2024-07-15T07:04:18.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:17 smithi089 bash[79810]: cluster 2024-07-15T07:04:17.323554+0000 mon.a (mon.0) 560 : cluster [WRN] Health check update: Degraded data redundancy: 14/72 objects degraded (19.444%), 9 pgs degraded (PG_DEGRADED) 2024-07-15T07:04:18.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:17 smithi089 bash[79810]: cluster 2024-07-15T07:04:17.883186+0000 mon.a (mon.0) 561 : cluster [DBG] osdmap e112: 8 total, 8 up, 8 in 2024-07-15T07:04:18.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:17 smithi089 bash[82202]: cluster 2024-07-15T07:04:16.873009+0000 mgr.y (mgr.25244) 306 : cluster [DBG] pgmap v250: 73 pgs: 3 peering, 6 active+clean+remapped, 19 active+undersized, 9 active+undersized+degraded, 36 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 14/72 objects degraded (19.444%); 1/72 objects misplaced (1.389%) 2024-07-15T07:04:18.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:17 smithi089 bash[82202]: cluster 2024-07-15T07:04:16.888503+0000 mon.a (mon.0) 559 : cluster [DBG] osdmap e111: 8 total, 8 up, 8 in 2024-07-15T07:04:18.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:17 smithi089 bash[82202]: cluster 2024-07-15T07:04:17.323554+0000 mon.a (mon.0) 560 : cluster [WRN] Health check update: Degraded data redundancy: 14/72 objects degraded (19.444%), 9 pgs degraded (PG_DEGRADED) 2024-07-15T07:04:18.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:17 smithi089 bash[82202]: cluster 2024-07-15T07:04:17.883186+0000 mon.a (mon.0) 561 : cluster [DBG] osdmap e112: 8 total, 8 up, 8 in 2024-07-15T07:04:18.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:17 smithi134 bash[67624]: cluster 2024-07-15T07:04:16.873009+0000 mgr.y (mgr.25244) 306 : cluster [DBG] pgmap v250: 73 pgs: 3 peering, 6 active+clean+remapped, 19 active+undersized, 9 active+undersized+degraded, 36 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 14/72 objects degraded (19.444%); 1/72 objects misplaced (1.389%) 2024-07-15T07:04:18.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:17 smithi134 bash[67624]: cluster 2024-07-15T07:04:16.888503+0000 mon.a (mon.0) 559 : cluster [DBG] osdmap e111: 8 total, 8 up, 8 in 2024-07-15T07:04:18.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:17 smithi134 bash[67624]: cluster 2024-07-15T07:04:17.323554+0000 mon.a (mon.0) 560 : cluster [WRN] Health check update: Degraded data redundancy: 14/72 objects degraded (19.444%), 9 pgs degraded (PG_DEGRADED) 2024-07-15T07:04:18.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:17 smithi134 bash[67624]: cluster 2024-07-15T07:04:17.883186+0000 mon.a (mon.0) 561 : cluster [DBG] osdmap e112: 8 total, 8 up, 8 in 2024-07-15T07:04:18.680 INFO:teuthology.orchestra.run.smithi089.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-07-15T07:04:18.680 INFO:teuthology.orchestra.run.smithi089.stdout:alertmanager.a smithi089 *:9093,9094 running (8m) 34s ago 16m 14.1M - 0.25.0 c8568f914cd2 c476af8941dd 2024-07-15T07:04:18.680 INFO:teuthology.orchestra.run.smithi089.stdout:grafana.a smithi134 *:3000 running (8m) 4s ago 15m 75.5M - 9.4.7 954c08fa6188 ac957811e4af 2024-07-15T07:04:18.680 INFO:teuthology.orchestra.run.smithi089.stdout:mds.foo.smithi089.cczuoz smithi089 running (13m) 34s ago 13m 14.8M - 17.2.7-1102-g9a202024 280b03ed6869 a1f028849336 2024-07-15T07:04:18.680 INFO:teuthology.orchestra.run.smithi089.stdout:mds.foo.smithi134.wugtzh smithi134 running (13m) 4s ago 13m 17.2M - 17.2.7-1102-g9a202024 280b03ed6869 fb5bbeafa32b 2024-07-15T07:04:18.680 INFO:teuthology.orchestra.run.smithi089.stdout:mgr.x smithi134 *:8443,9283,8765 running (7m) 4s ago 24m 426M - 18.2.2-1609-g6ea7aaa9 0add76435ed9 842a5eaedc06 2024-07-15T07:04:18.680 INFO:teuthology.orchestra.run.smithi089.stdout:mgr.y smithi089 *:8443,9283,8765 running (8m) 34s ago 27m 482M - 18.2.2-1609-g6ea7aaa9 0add76435ed9 20c56e260e99 2024-07-15T07:04:18.680 INFO:teuthology.orchestra.run.smithi089.stdout:mon.a smithi089 running (7m) 34s ago 27m 48.0M 2048M 18.2.2-1609-g6ea7aaa9 0add76435ed9 b2d2728b9d78 2024-07-15T07:04:18.680 INFO:teuthology.orchestra.run.smithi089.stdout:mon.b smithi134 running (5m) 4s ago 25m 37.2M 2048M 18.2.2-1609-g6ea7aaa9 0add76435ed9 f157e925047f 2024-07-15T07:04:18.680 INFO:teuthology.orchestra.run.smithi089.stdout:mon.c smithi089 running (5m) 34s ago 24m 41.9M 2048M 18.2.2-1609-g6ea7aaa9 0add76435ed9 60c08b0bcea8 2024-07-15T07:04:18.681 INFO:teuthology.orchestra.run.smithi089.stdout:node-exporter.a smithi089 *:9100 running (8m) 34s ago 16m 8511k - 1.5.0 0da6a335fe13 5da5d79c182f 2024-07-15T07:04:18.681 INFO:teuthology.orchestra.run.smithi089.stdout:node-exporter.b smithi134 *:9100 running (8m) 4s ago 16m 8328k - 1.5.0 0da6a335fe13 9ace5df7006d 2024-07-15T07:04:18.681 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0 smithi089 running (3m) 34s ago 23m 49.2M 2524M 18.2.2-1609-g6ea7aaa9 0add76435ed9 62162797039a 2024-07-15T07:04:18.681 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1 smithi089 running (2m) 34s ago 22m 42.2M 2524M 18.2.2-1609-g6ea7aaa9 0add76435ed9 fa4122a785bf 2024-07-15T07:04:18.681 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2 smithi089 running (68s) 34s ago 22m 48.7M 2524M 18.2.2-1609-g6ea7aaa9 0add76435ed9 b09291d6c7bd 2024-07-15T07:04:18.681 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3 smithi089 running (38s) 34s ago 21m - 2524M 18.2.2-1609-g6ea7aaa9 0add76435ed9 92c510e458f9 2024-07-15T07:04:18.681 INFO:teuthology.orchestra.run.smithi089.stdout:osd.4 smithi134 running (9s) 4s ago 20m - 2524M 18.2.2-1609-g6ea7aaa9 0add76435ed9 f73fbb5bebc7 2024-07-15T07:04:18.681 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5 smithi134 running (19m) 4s ago 19m 57.4M 2524M 17.2.7-1102-g9a202024 280b03ed6869 61be14c58fc3 2024-07-15T07:04:18.681 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6 smithi134 running (18m) 4s ago 18m 56.3M 2524M 17.2.7-1102-g9a202024 280b03ed6869 f92c7dd84666 2024-07-15T07:04:18.681 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7 smithi134 running (17m) 4s ago 17m 56.7M 2524M 17.2.7-1102-g9a202024 280b03ed6869 e44d11319ff6 2024-07-15T07:04:18.681 INFO:teuthology.orchestra.run.smithi089.stdout:prometheus.a smithi134 *:9095 running (7m) 4s ago 16m 56.1M - 2.43.0 a07b618ecd1d 8873e527bacf 2024-07-15T07:04:19.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:18 smithi089 bash[79810]: audit 2024-07-15T07:04:18.664727+0000 mgr.y (mgr.25244) 307 : audit [DBG] from='client.34365 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T07:04:19.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:18 smithi089 bash[82202]: audit 2024-07-15T07:04:18.664727+0000 mgr.y (mgr.25244) 307 : audit [DBG] from='client.34365 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T07:04:19.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:18 smithi134 bash[67624]: audit 2024-07-15T07:04:18.664727+0000 mgr.y (mgr.25244) 307 : audit [DBG] from='client.34365 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-07-15T07:04:19.419 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -e sha1=6ea7aaa9f93d34dc4d21862afe00140dfddbd706 -- bash -c 'ceph versions' 2024-07-15T07:04:20.985 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:20 smithi134 bash[67624]: cluster 2024-07-15T07:04:18.874106+0000 mgr.y (mgr.25244) 308 : cluster [DBG] pgmap v253: 73 pgs: 3 peering, 6 active+clean+remapped, 16 active+undersized, 8 active+undersized+degraded, 40 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 13/72 objects degraded (18.056%); 1/72 objects misplaced (1.389%) 2024-07-15T07:04:20.985 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:20 smithi134 bash[67624]: audit 2024-07-15T07:04:19.800880+0000 mon.a (mon.0) 562 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:04:20.985 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:20 smithi134 bash[67624]: audit 2024-07-15T07:04:19.802777+0000 mon.a (mon.0) 563 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:04:21.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:20 smithi089 bash[79810]: cluster 2024-07-15T07:04:18.874106+0000 mgr.y (mgr.25244) 308 : cluster [DBG] pgmap v253: 73 pgs: 3 peering, 6 active+clean+remapped, 16 active+undersized, 8 active+undersized+degraded, 40 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 13/72 objects degraded (18.056%); 1/72 objects misplaced (1.389%) 2024-07-15T07:04:21.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:20 smithi089 bash[79810]: audit 2024-07-15T07:04:19.800880+0000 mon.a (mon.0) 562 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:04:21.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:20 smithi089 bash[79810]: audit 2024-07-15T07:04:19.802777+0000 mon.a (mon.0) 563 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:04:21.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:20 smithi089 bash[82202]: cluster 2024-07-15T07:04:18.874106+0000 mgr.y (mgr.25244) 308 : cluster [DBG] pgmap v253: 73 pgs: 3 peering, 6 active+clean+remapped, 16 active+undersized, 8 active+undersized+degraded, 40 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 13/72 objects degraded (18.056%); 1/72 objects misplaced (1.389%) 2024-07-15T07:04:21.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:20 smithi089 bash[82202]: audit 2024-07-15T07:04:19.800880+0000 mon.a (mon.0) 562 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:04:21.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:20 smithi089 bash[82202]: audit 2024-07-15T07:04:19.802777+0000 mon.a (mon.0) 563 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:04:22.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:21 smithi089 bash[79810]: cluster 2024-07-15T07:04:21.798045+0000 mon.a (mon.0) 564 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 13/72 objects degraded (18.056%), 8 pgs degraded) 2024-07-15T07:04:22.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:21 smithi089 bash[82202]: cluster 2024-07-15T07:04:21.798045+0000 mon.a (mon.0) 564 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 13/72 objects degraded (18.056%), 8 pgs degraded) 2024-07-15T07:04:22.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:21 smithi134 bash[67624]: cluster 2024-07-15T07:04:21.798045+0000 mon.a (mon.0) 564 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 13/72 objects degraded (18.056%), 8 pgs degraded) 2024-07-15T07:04:22.602 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:04:22 smithi134 bash[60729]: ts=2024-07-15T07:04:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T07:04:23.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:22 smithi134 bash[67624]: cluster 2024-07-15T07:04:20.875139+0000 mgr.y (mgr.25244) 309 : cluster [DBG] pgmap v254: 73 pgs: 3 peering, 70 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 11 B/s, 0 objects/s recovering 2024-07-15T07:04:23.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:22 smithi089 bash[79810]: cluster 2024-07-15T07:04:20.875139+0000 mgr.y (mgr.25244) 309 : cluster [DBG] pgmap v254: 73 pgs: 3 peering, 70 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 11 B/s, 0 objects/s recovering 2024-07-15T07:04:23.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:22 smithi089 bash[82202]: cluster 2024-07-15T07:04:20.875139+0000 mgr.y (mgr.25244) 309 : cluster [DBG] pgmap v254: 73 pgs: 3 peering, 70 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 11 B/s, 0 objects/s recovering 2024-07-15T07:04:24.207 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T07:04:24.621 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 07:04:24 smithi089 bash[72894]: ::ffff:172.21.15.134 - - [15/Jul/2024:07:04:24] "GET /metrics HTTP/1.1" 200 33836 "" "Prometheus/2.43.0" 2024-07-15T07:04:25.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:24 smithi089 bash[79810]: cluster 2024-07-15T07:04:22.875870+0000 mgr.y (mgr.25244) 310 : cluster [DBG] pgmap v255: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 9 B/s, 0 objects/s recovering 2024-07-15T07:04:25.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:24 smithi089 bash[79810]: audit 2024-07-15T07:04:24.340710+0000 mon.a (mon.0) 565 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:04:25.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:24 smithi089 bash[79810]: audit 2024-07-15T07:04:24.362114+0000 mon.a (mon.0) 566 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:04:25.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:24 smithi089 bash[79810]: audit 2024-07-15T07:04:24.364355+0000 mon.a (mon.0) 567 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T07:04:25.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:24 smithi089 bash[79810]: audit 2024-07-15T07:04:24.366059+0000 mon.a (mon.0) 568 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T07:04:25.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:24 smithi089 bash[79810]: audit 2024-07-15T07:04:24.377150+0000 mon.a (mon.0) 569 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:04:25.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:24 smithi089 bash[79810]: audit 2024-07-15T07:04:24.528418+0000 mon.a (mon.0) 570 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T07:04:25.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:24 smithi089 bash[79810]: audit 2024-07-15T07:04:24.531650+0000 mon.a (mon.0) 571 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T07:04:25.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:24 smithi089 bash[79810]: audit 2024-07-15T07:04:24.533077+0000 mon.a (mon.0) 572 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T07:04:25.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:24 smithi089 bash[79810]: audit 2024-07-15T07:04:24.542698+0000 mon.a (mon.0) 573 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:04:25.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:24 smithi089 bash[82202]: cluster 2024-07-15T07:04:22.875870+0000 mgr.y (mgr.25244) 310 : cluster [DBG] pgmap v255: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 9 B/s, 0 objects/s recovering 2024-07-15T07:04:25.123 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:24 smithi089 bash[82202]: audit 2024-07-15T07:04:24.340710+0000 mon.a (mon.0) 565 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:04:25.123 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:24 smithi089 bash[82202]: audit 2024-07-15T07:04:24.362114+0000 mon.a (mon.0) 566 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:04:25.123 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:24 smithi089 bash[82202]: audit 2024-07-15T07:04:24.364355+0000 mon.a (mon.0) 567 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T07:04:25.123 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:24 smithi089 bash[82202]: audit 2024-07-15T07:04:24.366059+0000 mon.a (mon.0) 568 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T07:04:25.123 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:24 smithi089 bash[82202]: audit 2024-07-15T07:04:24.377150+0000 mon.a (mon.0) 569 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:04:25.123 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:24 smithi089 bash[82202]: audit 2024-07-15T07:04:24.528418+0000 mon.a (mon.0) 570 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T07:04:25.123 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:24 smithi089 bash[82202]: audit 2024-07-15T07:04:24.531650+0000 mon.a (mon.0) 571 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T07:04:25.123 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:24 smithi089 bash[82202]: audit 2024-07-15T07:04:24.533077+0000 mon.a (mon.0) 572 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T07:04:25.123 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:24 smithi089 bash[82202]: audit 2024-07-15T07:04:24.542698+0000 mon.a (mon.0) 573 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:04:25.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:24 smithi134 bash[67624]: cluster 2024-07-15T07:04:22.875870+0000 mgr.y (mgr.25244) 310 : cluster [DBG] pgmap v255: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 9 B/s, 0 objects/s recovering 2024-07-15T07:04:25.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:24 smithi134 bash[67624]: audit 2024-07-15T07:04:24.340710+0000 mon.a (mon.0) 565 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:04:25.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:24 smithi134 bash[67624]: audit 2024-07-15T07:04:24.362114+0000 mon.a (mon.0) 566 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:04:25.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:24 smithi134 bash[67624]: audit 2024-07-15T07:04:24.364355+0000 mon.a (mon.0) 567 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T07:04:25.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:24 smithi134 bash[67624]: audit 2024-07-15T07:04:24.366059+0000 mon.a (mon.0) 568 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T07:04:25.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:24 smithi134 bash[67624]: audit 2024-07-15T07:04:24.377150+0000 mon.a (mon.0) 569 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:04:25.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:24 smithi134 bash[67624]: audit 2024-07-15T07:04:24.528418+0000 mon.a (mon.0) 570 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T07:04:25.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:24 smithi134 bash[67624]: audit 2024-07-15T07:04:24.531650+0000 mon.a (mon.0) 571 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T07:04:25.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:24 smithi134 bash[67624]: audit 2024-07-15T07:04:24.533077+0000 mon.a (mon.0) 572 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T07:04:25.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:24 smithi134 bash[67624]: audit 2024-07-15T07:04:24.542698+0000 mon.a (mon.0) 573 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:04:26.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:25 smithi134 bash[67624]: cluster 2024-07-15T07:04:24.876645+0000 mgr.y (mgr.25244) 311 : cluster [DBG] pgmap v256: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 8 B/s, 0 objects/s recovering 2024-07-15T07:04:26.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:25 smithi089 bash[79810]: cluster 2024-07-15T07:04:24.876645+0000 mgr.y (mgr.25244) 311 : cluster [DBG] pgmap v256: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 8 B/s, 0 objects/s recovering 2024-07-15T07:04:26.371 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:25 smithi089 bash[82202]: cluster 2024-07-15T07:04:24.876645+0000 mgr.y (mgr.25244) 311 : cluster [DBG] pgmap v256: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 8 B/s, 0 objects/s recovering 2024-07-15T07:04:28.065 INFO:teuthology.orchestra.run.smithi089.stdout:{ 2024-07-15T07:04:28.065 INFO:teuthology.orchestra.run.smithi089.stdout: "mon": { 2024-07-15T07:04:28.065 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 18.2.2-1609-g6ea7aaa9 (6ea7aaa9f93d34dc4d21862afe00140dfddbd706) reef (stable)": 3 2024-07-15T07:04:28.065 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-07-15T07:04:28.065 INFO:teuthology.orchestra.run.smithi089.stdout: "mgr": { 2024-07-15T07:04:28.065 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 18.2.2-1609-g6ea7aaa9 (6ea7aaa9f93d34dc4d21862afe00140dfddbd706) reef (stable)": 2 2024-07-15T07:04:28.065 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-07-15T07:04:28.065 INFO:teuthology.orchestra.run.smithi089.stdout: "osd": { 2024-07-15T07:04:28.065 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 17.2.7-1102-g9a202024 (9a2020246f85463d4fdb115c94aa40b463caa1d2) quincy (stable)": 3, 2024-07-15T07:04:28.065 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 18.2.2-1609-g6ea7aaa9 (6ea7aaa9f93d34dc4d21862afe00140dfddbd706) reef (stable)": 5 2024-07-15T07:04:28.065 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-07-15T07:04:28.065 INFO:teuthology.orchestra.run.smithi089.stdout: "mds": { 2024-07-15T07:04:28.066 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 17.2.7-1102-g9a202024 (9a2020246f85463d4fdb115c94aa40b463caa1d2) quincy (stable)": 2 2024-07-15T07:04:28.066 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-07-15T07:04:28.066 INFO:teuthology.orchestra.run.smithi089.stdout: "overall": { 2024-07-15T07:04:28.066 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 17.2.7-1102-g9a202024 (9a2020246f85463d4fdb115c94aa40b463caa1d2) quincy (stable)": 5, 2024-07-15T07:04:28.066 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 18.2.2-1609-g6ea7aaa9 (6ea7aaa9f93d34dc4d21862afe00140dfddbd706) reef (stable)": 10 2024-07-15T07:04:28.066 INFO:teuthology.orchestra.run.smithi089.stdout: } 2024-07-15T07:04:28.066 INFO:teuthology.orchestra.run.smithi089.stdout:} 2024-07-15T07:04:28.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:27 smithi134 bash[67624]: cluster 2024-07-15T07:04:26.877629+0000 mgr.y (mgr.25244) 312 : cluster [DBG] pgmap v257: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 7 B/s, 0 objects/s recovering 2024-07-15T07:04:28.329 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:27 smithi089 bash[79810]: cluster 2024-07-15T07:04:26.877629+0000 mgr.y (mgr.25244) 312 : cluster [DBG] pgmap v257: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 7 B/s, 0 objects/s recovering 2024-07-15T07:04:28.329 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:27 smithi089 bash[82202]: cluster 2024-07-15T07:04:26.877629+0000 mgr.y (mgr.25244) 312 : cluster [DBG] pgmap v257: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 7 B/s, 0 objects/s recovering 2024-07-15T07:04:28.820 DEBUG:teuthology.parallel:result is None 2024-07-15T07:04:28.820 INFO:teuthology.run_tasks:Running task print... 2024-07-15T07:04:28.834 INFO:teuthology.task.print:**** done upgrading first half of cluster **** 2024-07-15T07:04:28.834 INFO:teuthology.run_tasks:Running task ceph.healthy... 2024-07-15T07:04:28.881 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-07-15T07:04:28.881 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-07-15T07:04:28.881 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph mgr dump --format=json 2024-07-15T07:04:29.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:28 smithi089 bash[79810]: audit 2024-07-15T07:04:28.068059+0000 mon.a (mon.0) 574 : audit [DBG] from='client.? 172.21.15.89:0/1750915716' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-07-15T07:04:29.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:28 smithi089 bash[82202]: audit 2024-07-15T07:04:28.068059+0000 mon.a (mon.0) 574 : audit [DBG] from='client.? 172.21.15.89:0/1750915716' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-07-15T07:04:29.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:28 smithi134 bash[67624]: audit 2024-07-15T07:04:28.068059+0000 mon.a (mon.0) 574 : audit [DBG] from='client.? 172.21.15.89:0/1750915716' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-07-15T07:04:30.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:29 smithi134 bash[67624]: cluster 2024-07-15T07:04:28.878362+0000 mgr.y (mgr.25244) 313 : cluster [DBG] pgmap v258: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 6 B/s, 0 objects/s recovering 2024-07-15T07:04:30.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:29 smithi089 bash[79810]: cluster 2024-07-15T07:04:28.878362+0000 mgr.y (mgr.25244) 313 : cluster [DBG] pgmap v258: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 6 B/s, 0 objects/s recovering 2024-07-15T07:04:30.371 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:29 smithi089 bash[82202]: cluster 2024-07-15T07:04:28.878362+0000 mgr.y (mgr.25244) 313 : cluster [DBG] pgmap v258: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 6 B/s, 0 objects/s recovering 2024-07-15T07:04:31.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:30 smithi134 bash[67624]: audit 2024-07-15T07:04:29.990078+0000 mon.a (mon.0) 575 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:04:31.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:30 smithi089 bash[79810]: audit 2024-07-15T07:04:29.990078+0000 mon.a (mon.0) 575 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:04:31.371 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:30 smithi089 bash[82202]: audit 2024-07-15T07:04:29.990078+0000 mon.a (mon.0) 575 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:04:32.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:31 smithi134 bash[67624]: cluster 2024-07-15T07:04:30.879352+0000 mgr.y (mgr.25244) 314 : cluster [DBG] pgmap v259: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 5 B/s, 0 objects/s recovering 2024-07-15T07:04:32.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:31 smithi089 bash[79810]: cluster 2024-07-15T07:04:30.879352+0000 mgr.y (mgr.25244) 314 : cluster [DBG] pgmap v259: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 5 B/s, 0 objects/s recovering 2024-07-15T07:04:32.371 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:31 smithi089 bash[82202]: cluster 2024-07-15T07:04:30.879352+0000 mgr.y (mgr.25244) 314 : cluster [DBG] pgmap v259: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 5 B/s, 0 objects/s recovering 2024-07-15T07:04:32.805 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:04:32 smithi134 bash[60729]: ts=2024-07-15T07:04:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T07:04:33.666 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T07:04:34.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:34 smithi089 bash[79810]: cluster 2024-07-15T07:04:32.880066+0000 mgr.y (mgr.25244) 315 : cluster [DBG] pgmap v260: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:04:34.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:34 smithi134 bash[67624]: cluster 2024-07-15T07:04:32.880066+0000 mgr.y (mgr.25244) 315 : cluster [DBG] pgmap v260: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:04:34.371 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:34 smithi089 bash[82202]: cluster 2024-07-15T07:04:32.880066+0000 mgr.y (mgr.25244) 315 : cluster [DBG] pgmap v260: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:04:34.371 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 07:04:34 smithi089 bash[72894]: ::ffff:172.21.15.134 - - [15/Jul/2024:07:04:34] "GET /metrics HTTP/1.1" 200 33836 "" "Prometheus/2.43.0" 2024-07-15T07:04:36.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:35 smithi134 bash[67624]: audit 2024-07-15T07:04:34.801919+0000 mon.a (mon.0) 576 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:04:36.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:35 smithi134 bash[67624]: audit 2024-07-15T07:04:34.804002+0000 mon.a (mon.0) 577 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:04:36.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:35 smithi089 bash[79810]: audit 2024-07-15T07:04:34.801919+0000 mon.a (mon.0) 576 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:04:36.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:35 smithi089 bash[79810]: audit 2024-07-15T07:04:34.804002+0000 mon.a (mon.0) 577 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:04:36.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:35 smithi089 bash[82202]: audit 2024-07-15T07:04:34.801919+0000 mon.a (mon.0) 576 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:04:36.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:35 smithi089 bash[82202]: audit 2024-07-15T07:04:34.804002+0000 mon.a (mon.0) 577 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:04:37.097 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:36 smithi089 bash[82202]: cluster 2024-07-15T07:04:34.880752+0000 mgr.y (mgr.25244) 316 : cluster [DBG] pgmap v261: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:04:37.097 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:36 smithi089 bash[79810]: cluster 2024-07-15T07:04:34.880752+0000 mgr.y (mgr.25244) 316 : cluster [DBG] pgmap v261: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:04:37.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:36 smithi134 bash[67624]: cluster 2024-07-15T07:04:34.880752+0000 mgr.y (mgr.25244) 316 : cluster [DBG] pgmap v261: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:04:37.495 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-07-15T07:04:38.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:37 smithi089 bash[79810]: audit 2024-07-15T07:04:37.488356+0000 mon.a (mon.0) 578 : audit [DBG] from='client.? 172.21.15.89:0/3872536469' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-07-15T07:04:38.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:37 smithi089 bash[82202]: audit 2024-07-15T07:04:37.488356+0000 mon.a (mon.0) 578 : audit [DBG] from='client.? 172.21.15.89:0/3872536469' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-07-15T07:04:38.207 INFO:teuthology.orchestra.run.smithi089.stdout:{"epoch":40,"flags":0,"active_gid":25244,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6800","nonce":2770079897},{"type":"v1","addr":"172.21.15.89:6801","nonce":2770079897}]},"active_addr":"172.21.15.89:6801/2770079897","active_change":"2024-07-15T06:57:04.675507+0000","active_mgr_features":4540138322906710015,"available":true,"standbys":[{"gid":24772,"name":"x","mgr_features":4540138322906710015,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.0.0","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.0.0","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.89:8443/","prometheus":"http://172.21.15.89:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":79,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.89:0","nonce":3508588063}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.89:0","nonce":527189439}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.89:0","nonce":357092277}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.89:0","nonce":3361307014}]}]} 2024-07-15T07:04:38.211 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-07-15T07:04:38.211 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-07-15T07:04:38.211 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph osd dump --format=json 2024-07-15T07:04:38.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:37 smithi134 bash[67624]: audit 2024-07-15T07:04:37.488356+0000 mon.a (mon.0) 578 : audit [DBG] from='client.? 172.21.15.89:0/3872536469' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-07-15T07:04:39.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:38 smithi089 bash[79810]: cluster 2024-07-15T07:04:36.881736+0000 mgr.y (mgr.25244) 317 : cluster [DBG] pgmap v262: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:04:39.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:38 smithi089 bash[82202]: cluster 2024-07-15T07:04:36.881736+0000 mgr.y (mgr.25244) 317 : cluster [DBG] pgmap v262: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:04:39.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:38 smithi134 bash[67624]: cluster 2024-07-15T07:04:36.881736+0000 mgr.y (mgr.25244) 317 : cluster [DBG] pgmap v262: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:04:41.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:40 smithi089 bash[79810]: cluster 2024-07-15T07:04:38.882494+0000 mgr.y (mgr.25244) 318 : cluster [DBG] pgmap v263: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:04:41.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:40 smithi089 bash[82202]: cluster 2024-07-15T07:04:38.882494+0000 mgr.y (mgr.25244) 318 : cluster [DBG] pgmap v263: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:04:41.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:40 smithi134 bash[67624]: cluster 2024-07-15T07:04:38.882494+0000 mgr.y (mgr.25244) 318 : cluster [DBG] pgmap v263: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:04:42.805 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:04:42 smithi134 bash[60729]: ts=2024-07-15T07:04:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T07:04:42.996 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T07:04:43.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:42 smithi089 bash[79810]: cluster 2024-07-15T07:04:40.883516+0000 mgr.y (mgr.25244) 319 : cluster [DBG] pgmap v264: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:04:43.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:42 smithi089 bash[82202]: cluster 2024-07-15T07:04:40.883516+0000 mgr.y (mgr.25244) 319 : cluster [DBG] pgmap v264: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:04:43.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:42 smithi134 bash[67624]: cluster 2024-07-15T07:04:40.883516+0000 mgr.y (mgr.25244) 319 : cluster [DBG] pgmap v264: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:04:44.416 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 07:04:44 smithi089 bash[72894]: ::ffff:172.21.15.134 - - [15/Jul/2024:07:04:44] "GET /metrics HTTP/1.1" 200 33980 "" "Prometheus/2.43.0" 2024-07-15T07:04:45.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:44 smithi089 bash[79810]: cluster 2024-07-15T07:04:42.884223+0000 mgr.y (mgr.25244) 320 : cluster [DBG] pgmap v265: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:04:45.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:44 smithi089 bash[82202]: cluster 2024-07-15T07:04:42.884223+0000 mgr.y (mgr.25244) 320 : cluster [DBG] pgmap v265: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:04:45.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:44 smithi134 bash[67624]: cluster 2024-07-15T07:04:42.884223+0000 mgr.y (mgr.25244) 320 : cluster [DBG] pgmap v265: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:04:46.688 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-07-15T07:04:46.688 INFO:teuthology.orchestra.run.smithi089.stdout:{"epoch":112,"fsid":"75ba02bc-4274-11ef-bcad-c7b262605968","created":"2024-07-15T06:37:09.634877+0000","modified":"2024-07-15T07:04:17.874997+0000","last_up_change":"2024-07-15T07:04:15.868428+0000","last_in_change":"2024-07-15T06:46:19.019927+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":23,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":5,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-07-15T06:42:21.444347+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-07-15T06:48:27.276344+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-07-15T06:50:28.088431+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"66","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-07-15T06:50:28.643335+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"67","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.5,"score_stable":2.5,"optimal_score":1,"raw_score_acting":2.5,"raw_score_stable":2.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"79357114-251f-489d-bef5-f6ec37002212","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":79,"up_from":83,"up_thru":110,"down_at":80,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6802","nonce":1073465095},{"type":"v1","addr":"172.21.15.89:6803","nonce":1073465095}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6804","nonce":1073465095},{"type":"v1","addr":"172.21.15.89:6805","nonce":1073465095}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6808","nonce":1073465095},{"type":"v1","addr":"172.21.15.89:6809","nonce":1073465095}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6806","nonce":1073465095},{"type":"v1","addr":"172.21.15.89:6807","nonce":1073465095}]},"public_addr":"172.21.15.89:6803/1073465095","cluster_addr":"172.21.15.89:6805/1073465095","heartbeat_back_addr":"172.21.15.89:6809/1073465095","heartbeat_front_addr":"172.21.15.89:6807/1073465095","state":["exists","up"]},{"osd":1,"uuid":"d304859e-317a-414e-a16c-f097fc4b2702","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":84,"up_from":88,"up_thru":110,"down_at":85,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6810","nonce":539615257},{"type":"v1","addr":"172.21.15.89:6811","nonce":539615257}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6812","nonce":539615257},{"type":"v1","addr":"172.21.15.89:6813","nonce":539615257}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6816","nonce":539615257},{"type":"v1","addr":"172.21.15.89:6817","nonce":539615257}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6814","nonce":539615257},{"type":"v1","addr":"172.21.15.89:6815","nonce":539615257}]},"public_addr":"172.21.15.89:6811/539615257","cluster_addr":"172.21.15.89:6813/539615257","heartbeat_back_addr":"172.21.15.89:6817/539615257","heartbeat_front_addr":"172.21.15.89:6815/539615257","state":["exists","up"]},{"osd":2,"uuid":"60cde686-83bb-4ca9-a0f1-6668710e16f1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":89,"up_from":96,"up_thru":111,"down_at":90,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6818","nonce":443062433},{"type":"v1","addr":"172.21.15.89:6819","nonce":443062433}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6820","nonce":443062433},{"type":"v1","addr":"172.21.15.89:6821","nonce":443062433}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6824","nonce":443062433},{"type":"v1","addr":"172.21.15.89:6825","nonce":443062433}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6822","nonce":443062433},{"type":"v1","addr":"172.21.15.89:6823","nonce":443062433}]},"public_addr":"172.21.15.89:6819/443062433","cluster_addr":"172.21.15.89:6821/443062433","heartbeat_back_addr":"172.21.15.89:6825/443062433","heartbeat_front_addr":"172.21.15.89:6823/443062433","state":["exists","up"]},{"osd":3,"uuid":"8af52bbd-4bc9-4b13-b525-26c92e69c29f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":98,"up_from":102,"up_thru":110,"down_at":99,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6826","nonce":3416042041},{"type":"v1","addr":"172.21.15.89:6827","nonce":3416042041}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6828","nonce":3416042041},{"type":"v1","addr":"172.21.15.89:6829","nonce":3416042041}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6832","nonce":3416042041},{"type":"v1","addr":"172.21.15.89:6833","nonce":3416042041}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6830","nonce":3416042041},{"type":"v1","addr":"172.21.15.89:6831","nonce":3416042041}]},"public_addr":"172.21.15.89:6827/3416042041","cluster_addr":"172.21.15.89:6829/3416042041","heartbeat_back_addr":"172.21.15.89:6833/3416042041","heartbeat_front_addr":"172.21.15.89:6831/3416042041","state":["exists","up"]},{"osd":4,"uuid":"b97874a6-e72a-4068-9c71-a50b8790be2c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":30,"last_clean_end":103,"up_from":110,"up_thru":111,"down_at":104,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6800","nonce":1882540973},{"type":"v1","addr":"172.21.15.134:6801","nonce":1882540973}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6802","nonce":1882540973},{"type":"v1","addr":"172.21.15.134:6803","nonce":1882540973}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6806","nonce":1882540973},{"type":"v1","addr":"172.21.15.134:6807","nonce":1882540973}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6804","nonce":1882540973},{"type":"v1","addr":"172.21.15.134:6805","nonce":1882540973}]},"public_addr":"172.21.15.134:6801/1882540973","cluster_addr":"172.21.15.134:6803/1882540973","heartbeat_back_addr":"172.21.15.134:6807/1882540973","heartbeat_front_addr":"172.21.15.134:6805/1882540973","state":["exists","up"]},{"osd":5,"uuid":"a834c8ee-e034-423d-ab67-1ab33bc15a97","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":111,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6808","nonce":823577783},{"type":"v1","addr":"172.21.15.134:6809","nonce":823577783}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6810","nonce":823577783},{"type":"v1","addr":"172.21.15.134:6811","nonce":823577783}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6814","nonce":823577783},{"type":"v1","addr":"172.21.15.134:6815","nonce":823577783}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6812","nonce":823577783},{"type":"v1","addr":"172.21.15.134:6813","nonce":823577783}]},"public_addr":"172.21.15.134:6809/823577783","cluster_addr":"172.21.15.134:6811/823577783","heartbeat_back_addr":"172.21.15.134:6815/823577783","heartbeat_front_addr":"172.21.15.134:6813/823577783","state":["exists","up"]},{"osd":6,"uuid":"446cb8fa-8ddb-4f7a-a8a4-c7281e950f74","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":110,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6816","nonce":4010812142},{"type":"v1","addr":"172.21.15.134:6817","nonce":4010812142}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6818","nonce":4010812142},{"type":"v1","addr":"172.21.15.134:6819","nonce":4010812142}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6822","nonce":4010812142},{"type":"v1","addr":"172.21.15.134:6823","nonce":4010812142}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6820","nonce":4010812142},{"type":"v1","addr":"172.21.15.134:6821","nonce":4010812142}]},"public_addr":"172.21.15.134:6817/4010812142","cluster_addr":"172.21.15.134:6819/4010812142","heartbeat_back_addr":"172.21.15.134:6823/4010812142","heartbeat_front_addr":"172.21.15.134:6821/4010812142","state":["exists","up"]},{"osd":7,"uuid":"87f3a30f-2935-4b92-af37-bd866696838f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":110,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6824","nonce":3883480074},{"type":"v1","addr":"172.21.15.134:6825","nonce":3883480074}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6826","nonce":3883480074},{"type":"v1","addr":"172.21.15.134:6827","nonce":3883480074}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6830","nonce":3883480074},{"type":"v1","addr":"172.21.15.134:6831","nonce":3883480074}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6828","nonce":3883480074},{"type":"v1","addr":"172.21.15.134:6829","nonce":3883480074}]},"public_addr":"172.21.15.134:6825/3883480074","cluster_addr":"172.21.15.134:6827/3883480074","heartbeat_back_addr":"172.21.15.134:6831/3883480074","heartbeat_front_addr":"172.21.15.134:6829/3883480074","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-07-15T06:40:36.567653+0000","dead_epoch":79},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-07-15T06:41:26.849575+0000","dead_epoch":84},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-07-15T06:42:18.731018+0000","dead_epoch":89},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-07-15T06:43:11.604111+0000","dead_epoch":98},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-07-15T06:44:03.847505+0000","dead_epoch":103},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-15T06:44:56.895305+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-15T06:45:50.039730+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-15T06:46:43.462786+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"3.5","mappings":[{"from":3,"to":4}]},{"pgid":"3.d","mappings":[{"from":7,"to":4}]},{"pgid":"4.4","mappings":[{"from":1,"to":5}]},{"pgid":"4.a","mappings":[{"from":1,"to":2}]},{"pgid":"4.e","mappings":[{"from":1,"to":0}]},{"pgid":"4.f","mappings":[{"from":1,"to":2}]},{"pgid":"4.11","mappings":[{"from":3,"to":4}]},{"pgid":"4.13","mappings":[{"from":1,"to":2}]},{"pgid":"4.14","mappings":[{"from":3,"to":2},{"from":1,"to":4}]},{"pgid":"4.17","mappings":[{"from":1,"to":0},{"from":3,"to":2}]},{"pgid":"4.1b","mappings":[{"from":3,"to":5}]},{"pgid":"4.1c","mappings":[{"from":1,"to":5}]},{"pgid":"4.1e","mappings":[{"from":3,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.89:6800/2527934197":"2024-07-16T06:55:00.492512+0000","172.21.15.134:6835/653500658":"2024-07-16T06:56:02.924271+0000","172.21.15.89:6801/2919301358":"2024-07-16T06:38:06.841135+0000","172.21.15.134:0/986904307":"2024-07-16T06:56:02.924271+0000","172.21.15.89:0/2281831659":"2024-07-16T06:55:00.492512+0000","172.21.15.89:0/1054475873":"2024-07-16T06:47:44.086461+0000","172.21.15.89:0/3680199495":"2024-07-16T06:47:44.086461+0000","172.21.15.134:0/4259446588":"2024-07-16T06:56:02.924271+0000","172.21.15.89:6801/153135008":"2024-07-16T06:57:04.674930+0000","172.21.15.89:0/1034002740":"2024-07-16T06:55:00.492512+0000","172.21.15.89:6800/265242087":"2024-07-16T06:47:44.086461+0000","172.21.15.89:0/1249162602":"2024-07-16T06:55:00.492512+0000","172.21.15.89:6801/2189595088":"2024-07-16T06:37:35.981306+0000","172.21.15.89:0/1672418864":"2024-07-16T06:38:06.841135+0000","172.21.15.89:0/889381338":"2024-07-16T06:38:06.841135+0000","172.21.15.89:0/1750227302":"2024-07-16T06:38:06.841135+0000","172.21.15.89:0/1877203535":"2024-07-16T06:37:35.981306+0000","172.21.15.89:0/1228522973":"2024-07-16T06:37:35.981306+0000","172.21.15.134:0/425069610":"2024-07-16T06:56:02.924271+0000","172.21.15.89:6801/265242087":"2024-07-16T06:47:44.086461+0000","172.21.15.134:0/2913422706":"2024-07-16T06:56:02.924271+0000","172.21.15.89:6801/2527934197":"2024-07-16T06:55:00.492512+0000","172.21.15.89:0/4152759072":"2024-07-16T06:57:04.674930+0000","172.21.15.134:6834/653500658":"2024-07-16T06:56:02.924271+0000","172.21.15.89:0/164236518":"2024-07-16T06:57:04.674930+0000","172.21.15.134:0/3559698020":"2024-07-16T06:56:02.924271+0000","172.21.15.89:6800/153135008":"2024-07-16T06:57:04.674930+0000","172.21.15.89:0/3524047244":"2024-07-16T06:55:00.492512+0000","172.21.15.89:0/980843739":"2024-07-16T06:57:04.674930+0000","172.21.15.89:0/2127823469":"2024-07-16T06:37:35.981306+0000","172.21.15.89:6834/4041986456":"2024-07-16T06:50:36.703871+0000","172.21.15.89:6835/4041986456":"2024-07-16T06:50:36.703871+0000","172.21.15.89:6800/2919301358":"2024-07-16T06:38:06.841135+0000","172.21.15.89:0/3465936199":"2024-07-16T06:47:44.086461+0000","172.21.15.89:0/2203949092":"2024-07-16T06:57:04.674930+0000","172.21.15.89:0/2377081311":"2024-07-16T06:57:04.674930+0000","172.21.15.89:6800/2189595088":"2024-07-16T06:37:35.981306+0000","172.21.15.89:0/2276119192":"2024-07-16T06:57:04.674930+0000","172.21.15.89:0/1100221668":"2024-07-16T06:47:44.086461+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":5,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-07-15T07:04:47.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:46 smithi089 bash[79810]: cluster 2024-07-15T07:04:44.884934+0000 mgr.y (mgr.25244) 321 : cluster [DBG] pgmap v266: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:04:47.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:46 smithi089 bash[79810]: audit 2024-07-15T07:04:46.689724+0000 mon.c (mon.2) 15 : audit [DBG] from='client.? 172.21.15.89:0/2838444746' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-07-15T07:04:47.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:46 smithi089 bash[82202]: cluster 2024-07-15T07:04:44.884934+0000 mgr.y (mgr.25244) 321 : cluster [DBG] pgmap v266: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:04:47.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:46 smithi089 bash[82202]: audit 2024-07-15T07:04:46.689724+0000 mon.c (mon.2) 15 : audit [DBG] from='client.? 172.21.15.89:0/2838444746' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-07-15T07:04:47.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:46 smithi134 bash[67624]: cluster 2024-07-15T07:04:44.884934+0000 mgr.y (mgr.25244) 321 : cluster [DBG] pgmap v266: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:04:47.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:46 smithi134 bash[67624]: audit 2024-07-15T07:04:46.689724+0000 mon.c (mon.2) 15 : audit [DBG] from='client.? 172.21.15.89:0/2838444746' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-07-15T07:04:47.471 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-07-15T07:04:47.472 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph osd dump --format=json 2024-07-15T07:04:49.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:48 smithi089 bash[79810]: cluster 2024-07-15T07:04:46.886070+0000 mgr.y (mgr.25244) 322 : cluster [DBG] pgmap v267: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:04:49.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:48 smithi089 bash[82202]: cluster 2024-07-15T07:04:46.886070+0000 mgr.y (mgr.25244) 322 : cluster [DBG] pgmap v267: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:04:49.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:48 smithi134 bash[67624]: cluster 2024-07-15T07:04:46.886070+0000 mgr.y (mgr.25244) 322 : cluster [DBG] pgmap v267: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:04:50.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:49 smithi089 bash[82202]: audit 2024-07-15T07:04:49.795353+0000 mon.a (mon.0) 579 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:04:50.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:49 smithi089 bash[79810]: audit 2024-07-15T07:04:49.795353+0000 mon.a (mon.0) 579 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:04:50.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:49 smithi134 bash[67624]: audit 2024-07-15T07:04:49.795353+0000 mon.a (mon.0) 579 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:04:51.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:50 smithi089 bash[79810]: cluster 2024-07-15T07:04:48.886930+0000 mgr.y (mgr.25244) 323 : cluster [DBG] pgmap v268: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:04:51.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:50 smithi089 bash[82202]: cluster 2024-07-15T07:04:48.886930+0000 mgr.y (mgr.25244) 323 : cluster [DBG] pgmap v268: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:04:51.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:50 smithi134 bash[67624]: cluster 2024-07-15T07:04:48.886930+0000 mgr.y (mgr.25244) 323 : cluster [DBG] pgmap v268: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:04:52.255 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T07:04:52.805 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:04:52 smithi134 bash[60729]: ts=2024-07-15T07:04:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T07:04:53.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:52 smithi089 bash[79810]: cluster 2024-07-15T07:04:50.887956+0000 mgr.y (mgr.25244) 324 : cluster [DBG] pgmap v269: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:04:53.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:52 smithi089 bash[82202]: cluster 2024-07-15T07:04:50.887956+0000 mgr.y (mgr.25244) 324 : cluster [DBG] pgmap v269: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:04:53.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:52 smithi134 bash[67624]: cluster 2024-07-15T07:04:50.887956+0000 mgr.y (mgr.25244) 324 : cluster [DBG] pgmap v269: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:04:54.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:53 smithi134 bash[67624]: cluster 2024-07-15T07:04:52.888605+0000 mgr.y (mgr.25244) 325 : cluster [DBG] pgmap v270: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:04:54.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:53 smithi089 bash[79810]: cluster 2024-07-15T07:04:52.888605+0000 mgr.y (mgr.25244) 325 : cluster [DBG] pgmap v270: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:04:54.372 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 07:04:54 smithi089 bash[72894]: ::ffff:172.21.15.134 - - [15/Jul/2024:07:04:54] "GET /metrics HTTP/1.1" 200 33981 "" "Prometheus/2.43.0" 2024-07-15T07:04:54.372 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:53 smithi089 bash[82202]: cluster 2024-07-15T07:04:52.888605+0000 mgr.y (mgr.25244) 325 : cluster [DBG] pgmap v270: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:04:56.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:55 smithi089 bash[79810]: cluster 2024-07-15T07:04:54.889312+0000 mgr.y (mgr.25244) 326 : cluster [DBG] pgmap v271: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:04:56.182 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:55 smithi089 bash[82202]: cluster 2024-07-15T07:04:54.889312+0000 mgr.y (mgr.25244) 326 : cluster [DBG] pgmap v271: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:04:56.183 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-07-15T07:04:56.183 INFO:teuthology.orchestra.run.smithi089.stdout:{"epoch":112,"fsid":"75ba02bc-4274-11ef-bcad-c7b262605968","created":"2024-07-15T06:37:09.634877+0000","modified":"2024-07-15T07:04:17.874997+0000","last_up_change":"2024-07-15T07:04:15.868428+0000","last_in_change":"2024-07-15T06:46:19.019927+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":23,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":5,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-07-15T06:42:21.444347+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-07-15T06:48:27.276344+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-07-15T06:50:28.088431+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"66","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-07-15T06:50:28.643335+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"67","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.5,"score_stable":2.5,"optimal_score":1,"raw_score_acting":2.5,"raw_score_stable":2.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"79357114-251f-489d-bef5-f6ec37002212","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":79,"up_from":83,"up_thru":110,"down_at":80,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6802","nonce":1073465095},{"type":"v1","addr":"172.21.15.89:6803","nonce":1073465095}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6804","nonce":1073465095},{"type":"v1","addr":"172.21.15.89:6805","nonce":1073465095}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6808","nonce":1073465095},{"type":"v1","addr":"172.21.15.89:6809","nonce":1073465095}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6806","nonce":1073465095},{"type":"v1","addr":"172.21.15.89:6807","nonce":1073465095}]},"public_addr":"172.21.15.89:6803/1073465095","cluster_addr":"172.21.15.89:6805/1073465095","heartbeat_back_addr":"172.21.15.89:6809/1073465095","heartbeat_front_addr":"172.21.15.89:6807/1073465095","state":["exists","up"]},{"osd":1,"uuid":"d304859e-317a-414e-a16c-f097fc4b2702","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":84,"up_from":88,"up_thru":110,"down_at":85,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6810","nonce":539615257},{"type":"v1","addr":"172.21.15.89:6811","nonce":539615257}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6812","nonce":539615257},{"type":"v1","addr":"172.21.15.89:6813","nonce":539615257}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6816","nonce":539615257},{"type":"v1","addr":"172.21.15.89:6817","nonce":539615257}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6814","nonce":539615257},{"type":"v1","addr":"172.21.15.89:6815","nonce":539615257}]},"public_addr":"172.21.15.89:6811/539615257","cluster_addr":"172.21.15.89:6813/539615257","heartbeat_back_addr":"172.21.15.89:6817/539615257","heartbeat_front_addr":"172.21.15.89:6815/539615257","state":["exists","up"]},{"osd":2,"uuid":"60cde686-83bb-4ca9-a0f1-6668710e16f1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":89,"up_from":96,"up_thru":111,"down_at":90,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6818","nonce":443062433},{"type":"v1","addr":"172.21.15.89:6819","nonce":443062433}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6820","nonce":443062433},{"type":"v1","addr":"172.21.15.89:6821","nonce":443062433}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6824","nonce":443062433},{"type":"v1","addr":"172.21.15.89:6825","nonce":443062433}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6822","nonce":443062433},{"type":"v1","addr":"172.21.15.89:6823","nonce":443062433}]},"public_addr":"172.21.15.89:6819/443062433","cluster_addr":"172.21.15.89:6821/443062433","heartbeat_back_addr":"172.21.15.89:6825/443062433","heartbeat_front_addr":"172.21.15.89:6823/443062433","state":["exists","up"]},{"osd":3,"uuid":"8af52bbd-4bc9-4b13-b525-26c92e69c29f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":98,"up_from":102,"up_thru":110,"down_at":99,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6826","nonce":3416042041},{"type":"v1","addr":"172.21.15.89:6827","nonce":3416042041}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6828","nonce":3416042041},{"type":"v1","addr":"172.21.15.89:6829","nonce":3416042041}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6832","nonce":3416042041},{"type":"v1","addr":"172.21.15.89:6833","nonce":3416042041}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6830","nonce":3416042041},{"type":"v1","addr":"172.21.15.89:6831","nonce":3416042041}]},"public_addr":"172.21.15.89:6827/3416042041","cluster_addr":"172.21.15.89:6829/3416042041","heartbeat_back_addr":"172.21.15.89:6833/3416042041","heartbeat_front_addr":"172.21.15.89:6831/3416042041","state":["exists","up"]},{"osd":4,"uuid":"b97874a6-e72a-4068-9c71-a50b8790be2c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":30,"last_clean_end":103,"up_from":110,"up_thru":111,"down_at":104,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6800","nonce":1882540973},{"type":"v1","addr":"172.21.15.134:6801","nonce":1882540973}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6802","nonce":1882540973},{"type":"v1","addr":"172.21.15.134:6803","nonce":1882540973}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6806","nonce":1882540973},{"type":"v1","addr":"172.21.15.134:6807","nonce":1882540973}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6804","nonce":1882540973},{"type":"v1","addr":"172.21.15.134:6805","nonce":1882540973}]},"public_addr":"172.21.15.134:6801/1882540973","cluster_addr":"172.21.15.134:6803/1882540973","heartbeat_back_addr":"172.21.15.134:6807/1882540973","heartbeat_front_addr":"172.21.15.134:6805/1882540973","state":["exists","up"]},{"osd":5,"uuid":"a834c8ee-e034-423d-ab67-1ab33bc15a97","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":111,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6808","nonce":823577783},{"type":"v1","addr":"172.21.15.134:6809","nonce":823577783}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6810","nonce":823577783},{"type":"v1","addr":"172.21.15.134:6811","nonce":823577783}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6814","nonce":823577783},{"type":"v1","addr":"172.21.15.134:6815","nonce":823577783}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6812","nonce":823577783},{"type":"v1","addr":"172.21.15.134:6813","nonce":823577783}]},"public_addr":"172.21.15.134:6809/823577783","cluster_addr":"172.21.15.134:6811/823577783","heartbeat_back_addr":"172.21.15.134:6815/823577783","heartbeat_front_addr":"172.21.15.134:6813/823577783","state":["exists","up"]},{"osd":6,"uuid":"446cb8fa-8ddb-4f7a-a8a4-c7281e950f74","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":110,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6816","nonce":4010812142},{"type":"v1","addr":"172.21.15.134:6817","nonce":4010812142}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6818","nonce":4010812142},{"type":"v1","addr":"172.21.15.134:6819","nonce":4010812142}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6822","nonce":4010812142},{"type":"v1","addr":"172.21.15.134:6823","nonce":4010812142}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6820","nonce":4010812142},{"type":"v1","addr":"172.21.15.134:6821","nonce":4010812142}]},"public_addr":"172.21.15.134:6817/4010812142","cluster_addr":"172.21.15.134:6819/4010812142","heartbeat_back_addr":"172.21.15.134:6823/4010812142","heartbeat_front_addr":"172.21.15.134:6821/4010812142","state":["exists","up"]},{"osd":7,"uuid":"87f3a30f-2935-4b92-af37-bd866696838f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":110,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6824","nonce":3883480074},{"type":"v1","addr":"172.21.15.134:6825","nonce":3883480074}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6826","nonce":3883480074},{"type":"v1","addr":"172.21.15.134:6827","nonce":3883480074}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6830","nonce":3883480074},{"type":"v1","addr":"172.21.15.134:6831","nonce":3883480074}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6828","nonce":3883480074},{"type":"v1","addr":"172.21.15.134:6829","nonce":3883480074}]},"public_addr":"172.21.15.134:6825/3883480074","cluster_addr":"172.21.15.134:6827/3883480074","heartbeat_back_addr":"172.21.15.134:6831/3883480074","heartbeat_front_addr":"172.21.15.134:6829/3883480074","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-07-15T06:40:36.567653+0000","dead_epoch":79},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-07-15T06:41:26.849575+0000","dead_epoch":84},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-07-15T06:42:18.731018+0000","dead_epoch":89},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-07-15T06:43:11.604111+0000","dead_epoch":98},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-07-15T06:44:03.847505+0000","dead_epoch":103},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-15T06:44:56.895305+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-15T06:45:50.039730+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-15T06:46:43.462786+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"3.5","mappings":[{"from":3,"to":4}]},{"pgid":"3.d","mappings":[{"from":7,"to":4}]},{"pgid":"4.4","mappings":[{"from":1,"to":5}]},{"pgid":"4.a","mappings":[{"from":1,"to":2}]},{"pgid":"4.e","mappings":[{"from":1,"to":0}]},{"pgid":"4.f","mappings":[{"from":1,"to":2}]},{"pgid":"4.11","mappings":[{"from":3,"to":4}]},{"pgid":"4.13","mappings":[{"from":1,"to":2}]},{"pgid":"4.14","mappings":[{"from":3,"to":2},{"from":1,"to":4}]},{"pgid":"4.17","mappings":[{"from":1,"to":0},{"from":3,"to":2}]},{"pgid":"4.1b","mappings":[{"from":3,"to":5}]},{"pgid":"4.1c","mappings":[{"from":1,"to":5}]},{"pgid":"4.1e","mappings":[{"from":3,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.89:6800/2527934197":"2024-07-16T06:55:00.492512+0000","172.21.15.134:6835/653500658":"2024-07-16T06:56:02.924271+0000","172.21.15.89:6801/2919301358":"2024-07-16T06:38:06.841135+0000","172.21.15.134:0/986904307":"2024-07-16T06:56:02.924271+0000","172.21.15.89:0/2281831659":"2024-07-16T06:55:00.492512+0000","172.21.15.89:0/1054475873":"2024-07-16T06:47:44.086461+0000","172.21.15.89:0/3680199495":"2024-07-16T06:47:44.086461+0000","172.21.15.134:0/4259446588":"2024-07-16T06:56:02.924271+0000","172.21.15.89:6801/153135008":"2024-07-16T06:57:04.674930+0000","172.21.15.89:0/1034002740":"2024-07-16T06:55:00.492512+0000","172.21.15.89:6800/265242087":"2024-07-16T06:47:44.086461+0000","172.21.15.89:0/1249162602":"2024-07-16T06:55:00.492512+0000","172.21.15.89:6801/2189595088":"2024-07-16T06:37:35.981306+0000","172.21.15.89:0/1672418864":"2024-07-16T06:38:06.841135+0000","172.21.15.89:0/889381338":"2024-07-16T06:38:06.841135+0000","172.21.15.89:0/1750227302":"2024-07-16T06:38:06.841135+0000","172.21.15.89:0/1877203535":"2024-07-16T06:37:35.981306+0000","172.21.15.89:0/1228522973":"2024-07-16T06:37:35.981306+0000","172.21.15.134:0/425069610":"2024-07-16T06:56:02.924271+0000","172.21.15.89:6801/265242087":"2024-07-16T06:47:44.086461+0000","172.21.15.134:0/2913422706":"2024-07-16T06:56:02.924271+0000","172.21.15.89:6801/2527934197":"2024-07-16T06:55:00.492512+0000","172.21.15.89:0/4152759072":"2024-07-16T06:57:04.674930+0000","172.21.15.134:6834/653500658":"2024-07-16T06:56:02.924271+0000","172.21.15.89:0/164236518":"2024-07-16T06:57:04.674930+0000","172.21.15.134:0/3559698020":"2024-07-16T06:56:02.924271+0000","172.21.15.89:6800/153135008":"2024-07-16T06:57:04.674930+0000","172.21.15.89:0/3524047244":"2024-07-16T06:55:00.492512+0000","172.21.15.89:0/980843739":"2024-07-16T06:57:04.674930+0000","172.21.15.89:0/2127823469":"2024-07-16T06:37:35.981306+0000","172.21.15.89:6834/4041986456":"2024-07-16T06:50:36.703871+0000","172.21.15.89:6835/4041986456":"2024-07-16T06:50:36.703871+0000","172.21.15.89:6800/2919301358":"2024-07-16T06:38:06.841135+0000","172.21.15.89:0/3465936199":"2024-07-16T06:47:44.086461+0000","172.21.15.89:0/2203949092":"2024-07-16T06:57:04.674930+0000","172.21.15.89:0/2377081311":"2024-07-16T06:57:04.674930+0000","172.21.15.89:6800/2189595088":"2024-07-16T06:37:35.981306+0000","172.21.15.89:0/2276119192":"2024-07-16T06:57:04.674930+0000","172.21.15.89:0/1100221668":"2024-07-16T06:47:44.086461+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":5,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-07-15T07:04:56.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:55 smithi134 bash[67624]: cluster 2024-07-15T07:04:54.889312+0000 mgr.y (mgr.25244) 326 : cluster [DBG] pgmap v271: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:04:56.990 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-07-15T07:04:56.991 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-07-15T07:04:56.991 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-07-15T07:04:56.992 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-07-15T07:04:56.992 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-07-15T07:04:56.992 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-07-15T07:04:56.993 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-07-15T07:04:56.993 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-07-15T07:04:57.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:56 smithi134 bash[67624]: audit 2024-07-15T07:04:56.179009+0000 mon.c (mon.2) 16 : audit [DBG] from='client.? 172.21.15.89:0/4102852815' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-07-15T07:04:57.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:56 smithi089 bash[79810]: audit 2024-07-15T07:04:56.179009+0000 mon.c (mon.2) 16 : audit [DBG] from='client.? 172.21.15.89:0/4102852815' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-07-15T07:04:57.371 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:56 smithi089 bash[82202]: audit 2024-07-15T07:04:56.179009+0000 mon.c (mon.2) 16 : audit [DBG] from='client.? 172.21.15.89:0/4102852815' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-07-15T07:04:58.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:57 smithi134 bash[67624]: cluster 2024-07-15T07:04:56.890321+0000 mgr.y (mgr.25244) 327 : cluster [DBG] pgmap v272: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:04:58.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:57 smithi089 bash[79810]: cluster 2024-07-15T07:04:56.890321+0000 mgr.y (mgr.25244) 327 : cluster [DBG] pgmap v272: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:04:58.372 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:57 smithi089 bash[82202]: cluster 2024-07-15T07:04:56.890321+0000 mgr.y (mgr.25244) 327 : cluster [DBG] pgmap v272: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:05:00.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:04:59 smithi134 bash[67624]: cluster 2024-07-15T07:04:58.891030+0000 mgr.y (mgr.25244) 328 : cluster [DBG] pgmap v273: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:05:00.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:04:59 smithi089 bash[79810]: cluster 2024-07-15T07:04:58.891030+0000 mgr.y (mgr.25244) 328 : cluster [DBG] pgmap v273: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:05:00.371 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:04:59 smithi089 bash[82202]: cluster 2024-07-15T07:04:58.891030+0000 mgr.y (mgr.25244) 328 : cluster [DBG] pgmap v273: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:05:01.885 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T07:05:01.886 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T07:05:01.886 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T07:05:01.886 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T07:05:01.888 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T07:05:01.888 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T07:05:01.890 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T07:05:01.894 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T07:05:02.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:05:01 smithi089 bash[79810]: cluster 2024-07-15T07:05:00.891985+0000 mgr.y (mgr.25244) 329 : cluster [DBG] pgmap v274: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:05:02.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:05:01 smithi089 bash[82202]: cluster 2024-07-15T07:05:00.891985+0000 mgr.y (mgr.25244) 329 : cluster [DBG] pgmap v274: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:05:02.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:05:01 smithi134 bash[67624]: cluster 2024-07-15T07:05:00.891985+0000 mgr.y (mgr.25244) 329 : cluster [DBG] pgmap v274: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:05:02.805 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:05:02 smithi134 bash[60729]: ts=2024-07-15T07:05:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T07:05:04.085 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:05:03 smithi089 bash[79810]: cluster 2024-07-15T07:05:02.892726+0000 mgr.y (mgr.25244) 330 : cluster [DBG] pgmap v275: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:05:04.085 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:05:03 smithi089 bash[82202]: cluster 2024-07-15T07:05:02.892726+0000 mgr.y (mgr.25244) 330 : cluster [DBG] pgmap v275: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:05:04.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:05:03 smithi134 bash[67624]: cluster 2024-07-15T07:05:02.892726+0000 mgr.y (mgr.25244) 330 : cluster [DBG] pgmap v275: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:05:04.371 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 07:05:04 smithi089 bash[72894]: ::ffff:172.21.15.134 - - [15/Jul/2024:07:05:04] "GET /metrics HTTP/1.1" 200 33981 "" "Prometheus/2.43.0" 2024-07-15T07:05:05.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:05:04 smithi089 bash[79810]: audit 2024-07-15T07:05:04.795096+0000 mon.a (mon.0) 580 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:05:05.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:05:04 smithi089 bash[82202]: audit 2024-07-15T07:05:04.795096+0000 mon.a (mon.0) 580 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:05:05.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:05:04 smithi134 bash[67624]: audit 2024-07-15T07:05:04.795096+0000 mon.a (mon.0) 580 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:05:06.204 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:05:05 smithi089 bash[82202]: cluster 2024-07-15T07:05:04.893358+0000 mgr.y (mgr.25244) 331 : cluster [DBG] pgmap v276: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:05:06.204 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:05:05 smithi089 bash[82202]: audit 2024-07-15T07:05:05.829503+0000 mon.a (mon.0) 581 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "3.5"}]: dispatch 2024-07-15T07:05:06.204 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:05:05 smithi089 bash[82202]: audit 2024-07-15T07:05:05.829915+0000 mon.a (mon.0) 582 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "3.d"}]: dispatch 2024-07-15T07:05:06.204 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:05:05 smithi089 bash[79810]: cluster 2024-07-15T07:05:04.893358+0000 mgr.y (mgr.25244) 331 : cluster [DBG] pgmap v276: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:05:06.205 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:05:05 smithi089 bash[79810]: audit 2024-07-15T07:05:05.829503+0000 mon.a (mon.0) 581 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "3.5"}]: dispatch 2024-07-15T07:05:06.205 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:05:05 smithi089 bash[79810]: audit 2024-07-15T07:05:05.829915+0000 mon.a (mon.0) 582 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "3.d"}]: dispatch 2024-07-15T07:05:06.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:05:05 smithi134 bash[67624]: cluster 2024-07-15T07:05:04.893358+0000 mgr.y (mgr.25244) 331 : cluster [DBG] pgmap v276: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:05:06.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:05:05 smithi134 bash[67624]: audit 2024-07-15T07:05:05.829503+0000 mon.a (mon.0) 581 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "3.5"}]: dispatch 2024-07-15T07:05:06.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:05:05 smithi134 bash[67624]: audit 2024-07-15T07:05:05.829915+0000 mon.a (mon.0) 582 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "3.d"}]: dispatch 2024-07-15T07:05:07.284 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:05:06 smithi089 bash[82202]: audit 2024-07-15T07:05:05.967124+0000 mon.a (mon.0) 583 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd='[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "3.5"}]': finished 2024-07-15T07:05:07.284 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:05:06 smithi089 bash[82202]: audit 2024-07-15T07:05:05.967277+0000 mon.a (mon.0) 584 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd='[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "3.d"}]': finished 2024-07-15T07:05:07.284 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:05:06 smithi089 bash[82202]: cluster 2024-07-15T07:05:05.967439+0000 mon.a (mon.0) 585 : cluster [DBG] osdmap e113: 8 total, 8 up, 8 in 2024-07-15T07:05:07.284 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:05:06 smithi089 bash[79810]: audit 2024-07-15T07:05:05.967124+0000 mon.a (mon.0) 583 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd='[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "3.5"}]': finished 2024-07-15T07:05:07.285 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:05:06 smithi089 bash[79810]: audit 2024-07-15T07:05:05.967277+0000 mon.a (mon.0) 584 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd='[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "3.d"}]': finished 2024-07-15T07:05:07.285 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:05:06 smithi089 bash[79810]: cluster 2024-07-15T07:05:05.967439+0000 mon.a (mon.0) 585 : cluster [DBG] osdmap e113: 8 total, 8 up, 8 in 2024-07-15T07:05:07.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:05:06 smithi134 bash[67624]: audit 2024-07-15T07:05:05.967124+0000 mon.a (mon.0) 583 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd='[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "3.5"}]': finished 2024-07-15T07:05:07.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:05:06 smithi134 bash[67624]: audit 2024-07-15T07:05:05.967277+0000 mon.a (mon.0) 584 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd='[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "3.d"}]': finished 2024-07-15T07:05:07.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:05:06 smithi134 bash[67624]: cluster 2024-07-15T07:05:05.967439+0000 mon.a (mon.0) 585 : cluster [DBG] osdmap e113: 8 total, 8 up, 8 in 2024-07-15T07:05:08.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:05:07 smithi134 bash[67624]: cluster 2024-07-15T07:05:06.894047+0000 mgr.y (mgr.25244) 332 : cluster [DBG] pgmap v278: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:05:08.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:05:07 smithi134 bash[67624]: cluster 2024-07-15T07:05:06.975136+0000 mon.a (mon.0) 586 : cluster [DBG] osdmap e114: 8 total, 8 up, 8 in 2024-07-15T07:05:08.371 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:05:07 smithi089 bash[82202]: cluster 2024-07-15T07:05:06.894047+0000 mgr.y (mgr.25244) 332 : cluster [DBG] pgmap v278: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:05:08.372 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:05:07 smithi089 bash[82202]: cluster 2024-07-15T07:05:06.975136+0000 mon.a (mon.0) 586 : cluster [DBG] osdmap e114: 8 total, 8 up, 8 in 2024-07-15T07:05:08.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:05:07 smithi089 bash[79810]: cluster 2024-07-15T07:05:06.894047+0000 mgr.y (mgr.25244) 332 : cluster [DBG] pgmap v278: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:05:08.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:05:07 smithi089 bash[79810]: cluster 2024-07-15T07:05:06.975136+0000 mon.a (mon.0) 586 : cluster [DBG] osdmap e114: 8 total, 8 up, 8 in 2024-07-15T07:05:10.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:05:09 smithi134 bash[67624]: cluster 2024-07-15T07:05:08.894512+0000 mgr.y (mgr.25244) 333 : cluster [DBG] pgmap v280: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:05:10.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:05:09 smithi089 bash[79810]: cluster 2024-07-15T07:05:08.894512+0000 mgr.y (mgr.25244) 333 : cluster [DBG] pgmap v280: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:05:10.372 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:05:09 smithi089 bash[82202]: cluster 2024-07-15T07:05:08.894512+0000 mgr.y (mgr.25244) 333 : cluster [DBG] pgmap v280: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:05:12.305 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:05:11 smithi134 bash[67624]: cluster 2024-07-15T07:05:10.895465+0000 mgr.y (mgr.25244) 334 : cluster [DBG] pgmap v281: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 8 B/s, 0 objects/s recovering 2024-07-15T07:05:12.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:05:11 smithi089 bash[79810]: cluster 2024-07-15T07:05:10.895465+0000 mgr.y (mgr.25244) 334 : cluster [DBG] pgmap v281: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 8 B/s, 0 objects/s recovering 2024-07-15T07:05:12.372 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:05:11 smithi089 bash[82202]: cluster 2024-07-15T07:05:10.895465+0000 mgr.y (mgr.25244) 334 : cluster [DBG] pgmap v281: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 8 B/s, 0 objects/s recovering 2024-07-15T07:05:12.805 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:05:12 smithi134 bash[60729]: ts=2024-07-15T07:05:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T07:05:14.339 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 07:05:14 smithi089 bash[72894]: ::ffff:172.21.15.134 - - [15/Jul/2024:07:05:14] "GET /metrics HTTP/1.1" 200 33979 "" "Prometheus/2.43.0" 2024-07-15T07:05:14.657 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:05:14 smithi089 bash[79810]: cluster 2024-07-15T07:05:12.896252+0000 mgr.y (mgr.25244) 335 : cluster [DBG] pgmap v282: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 8 B/s, 0 objects/s recovering 2024-07-15T07:05:14.657 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:05:14 smithi089 bash[82202]: cluster 2024-07-15T07:05:12.896252+0000 mgr.y (mgr.25244) 335 : cluster [DBG] pgmap v282: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 8 B/s, 0 objects/s recovering 2024-07-15T07:05:14.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:05:14 smithi134 bash[67624]: cluster 2024-07-15T07:05:12.896252+0000 mgr.y (mgr.25244) 335 : cluster [DBG] pgmap v282: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 8 B/s, 0 objects/s recovering 2024-07-15T07:05:16.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:05:16 smithi089 bash[79810]: cluster 2024-07-15T07:05:14.897029+0000 mgr.y (mgr.25244) 336 : cluster [DBG] pgmap v283: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 7 B/s, 0 objects/s recovering 2024-07-15T07:05:16.626 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:05:16 smithi089 bash[82202]: cluster 2024-07-15T07:05:14.897029+0000 mgr.y (mgr.25244) 336 : cluster [DBG] pgmap v283: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 7 B/s, 0 objects/s recovering 2024-07-15T07:05:16.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:05:16 smithi134 bash[67624]: cluster 2024-07-15T07:05:14.897029+0000 mgr.y (mgr.25244) 336 : cluster [DBG] pgmap v283: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 7 B/s, 0 objects/s recovering 2024-07-15T07:05:17.961 INFO:teuthology.orchestra.run.smithi089.stdout:356482285618 2024-07-15T07:05:17.962 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-07-15T07:05:18.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:05:18 smithi089 bash[82202]: cluster 2024-07-15T07:05:16.897592+0000 mgr.y (mgr.25244) 337 : cluster [DBG] pgmap v284: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 6 B/s, 0 objects/s recovering 2024-07-15T07:05:18.622 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:05:18 smithi089 bash[79810]: cluster 2024-07-15T07:05:16.897592+0000 mgr.y (mgr.25244) 337 : cluster [DBG] pgmap v284: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 6 B/s, 0 objects/s recovering 2024-07-15T07:05:18.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:05:18 smithi134 bash[67624]: cluster 2024-07-15T07:05:16.897592+0000 mgr.y (mgr.25244) 337 : cluster [DBG] pgmap v284: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 6 B/s, 0 objects/s recovering 2024-07-15T07:05:19.549 INFO:teuthology.orchestra.run.smithi089.stdout:163208757495 2024-07-15T07:05:19.549 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-07-15T07:05:20.316 INFO:teuthology.orchestra.run.smithi089.stdout:412316860442 2024-07-15T07:05:20.317 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-07-15T07:05:20.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:05:20 smithi089 bash[79810]: cluster 2024-07-15T07:05:18.898203+0000 mgr.y (mgr.25244) 338 : cluster [DBG] pgmap v285: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 5 B/s, 0 objects/s recovering 2024-07-15T07:05:20.622 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:05:20 smithi089 bash[79810]: audit 2024-07-15T07:05:19.795909+0000 mon.a (mon.0) 587 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:05:20.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:05:20 smithi089 bash[82202]: cluster 2024-07-15T07:05:18.898203+0000 mgr.y (mgr.25244) 338 : cluster [DBG] pgmap v285: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 5 B/s, 0 objects/s recovering 2024-07-15T07:05:20.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:05:20 smithi089 bash[82202]: audit 2024-07-15T07:05:19.795909+0000 mon.a (mon.0) 587 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:05:20.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:05:20 smithi134 bash[67624]: cluster 2024-07-15T07:05:18.898203+0000 mgr.y (mgr.25244) 338 : cluster [DBG] pgmap v285: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 5 B/s, 0 objects/s recovering 2024-07-15T07:05:20.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:05:20 smithi134 bash[67624]: audit 2024-07-15T07:05:19.795909+0000 mon.a (mon.0) 587 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:05:21.936 INFO:teuthology.orchestra.run.smithi089.stdout:438086664212 2024-07-15T07:05:21.937 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-07-15T07:05:22.415 INFO:teuthology.orchestra.run.smithi089.stdout:472446402575 2024-07-15T07:05:22.415 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-07-15T07:05:22.622 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:05:22 smithi089 bash[79810]: cluster 2024-07-15T07:05:20.899170+0000 mgr.y (mgr.25244) 339 : cluster [DBG] pgmap v286: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 5 B/s, 0 objects/s recovering 2024-07-15T07:05:22.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:05:22 smithi089 bash[82202]: cluster 2024-07-15T07:05:20.899170+0000 mgr.y (mgr.25244) 339 : cluster [DBG] pgmap v286: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 5 B/s, 0 objects/s recovering 2024-07-15T07:05:22.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:05:22 smithi134 bash[67624]: cluster 2024-07-15T07:05:20.899170+0000 mgr.y (mgr.25244) 339 : cluster [DBG] pgmap v286: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 5 B/s, 0 objects/s recovering 2024-07-15T07:05:22.806 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:05:22 smithi134 bash[60729]: ts=2024-07-15T07:05:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T07:05:23.519 INFO:teuthology.orchestra.run.smithi089.stdout:231928234212 2024-07-15T07:05:23.519 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-07-15T07:05:23.605 INFO:teuthology.orchestra.run.smithi089.stdout:377957122092 2024-07-15T07:05:23.606 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-07-15T07:05:23.648 INFO:teuthology.orchestra.run.smithi089.stdout:197568495854 2024-07-15T07:05:23.649 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-07-15T07:05:24.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:05:24 smithi089 bash[79810]: cluster 2024-07-15T07:05:22.899970+0000 mgr.y (mgr.25244) 340 : cluster [DBG] pgmap v287: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:05:24.622 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 07:05:24 smithi089 bash[72894]: ::ffff:172.21.15.134 - - [15/Jul/2024:07:05:24] "GET /metrics HTTP/1.1" 200 33979 "" "Prometheus/2.43.0" 2024-07-15T07:05:24.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:05:24 smithi089 bash[82202]: cluster 2024-07-15T07:05:22.899970+0000 mgr.y (mgr.25244) 340 : cluster [DBG] pgmap v287: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:05:24.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:05:24 smithi134 bash[67624]: cluster 2024-07-15T07:05:22.899970+0000 mgr.y (mgr.25244) 340 : cluster [DBG] pgmap v287: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:05:25.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:05:25 smithi089 bash[79810]: audit 2024-07-15T07:05:24.654336+0000 mon.a (mon.0) 588 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T07:05:25.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:05:25 smithi089 bash[82202]: audit 2024-07-15T07:05:24.654336+0000 mon.a (mon.0) 588 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T07:05:25.785 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T07:05:25.786 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T07:05:25.786 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T07:05:25.786 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T07:05:25.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:05:25 smithi134 bash[67624]: audit 2024-07-15T07:05:24.654336+0000 mon.a (mon.0) 588 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T07:05:26.622 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:05:26 smithi089 bash[79810]: cluster 2024-07-15T07:05:24.900607+0000 mgr.y (mgr.25244) 341 : cluster [DBG] pgmap v288: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:05:26.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:05:26 smithi089 bash[82202]: cluster 2024-07-15T07:05:24.900607+0000 mgr.y (mgr.25244) 341 : cluster [DBG] pgmap v288: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:05:26.802 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T07:05:26.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:05:26 smithi134 bash[67624]: cluster 2024-07-15T07:05:24.900607+0000 mgr.y (mgr.25244) 341 : cluster [DBG] pgmap v288: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:05:27.825 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T07:05:27.826 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T07:05:27.827 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T07:05:28.375 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:05:28 smithi089 bash[79810]: cluster 2024-07-15T07:05:26.901686+0000 mgr.y (mgr.25244) 342 : cluster [DBG] pgmap v289: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:05:28.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:05:28 smithi134 bash[67624]: cluster 2024-07-15T07:05:26.901686+0000 mgr.y (mgr.25244) 342 : cluster [DBG] pgmap v289: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:05:28.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:05:28 smithi089 bash[82202]: cluster 2024-07-15T07:05:26.901686+0000 mgr.y (mgr.25244) 342 : cluster [DBG] pgmap v289: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:05:30.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:05:30 smithi134 bash[67624]: cluster 2024-07-15T07:05:28.902132+0000 mgr.y (mgr.25244) 343 : cluster [DBG] pgmap v290: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:05:30.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:05:30 smithi134 bash[67624]: audit 2024-07-15T07:05:29.542590+0000 mon.a (mon.0) 589 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:05:30.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:05:30 smithi134 bash[67624]: audit 2024-07-15T07:05:29.547117+0000 mon.a (mon.0) 590 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:05:30.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:05:30 smithi134 bash[67624]: audit 2024-07-15T07:05:30.456514+0000 mon.a (mon.0) 591 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:05:30.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:05:30 smithi134 bash[67624]: audit 2024-07-15T07:05:30.466711+0000 mon.a (mon.0) 592 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:05:30.860 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:05:30 smithi089 bash[79810]: cluster 2024-07-15T07:05:28.902132+0000 mgr.y (mgr.25244) 343 : cluster [DBG] pgmap v290: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:05:30.860 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:05:30 smithi089 bash[79810]: audit 2024-07-15T07:05:29.542590+0000 mon.a (mon.0) 589 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:05:30.860 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:05:30 smithi089 bash[79810]: audit 2024-07-15T07:05:29.547117+0000 mon.a (mon.0) 590 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:05:30.860 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:05:30 smithi089 bash[79810]: audit 2024-07-15T07:05:30.456514+0000 mon.a (mon.0) 591 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:05:30.860 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:05:30 smithi089 bash[79810]: audit 2024-07-15T07:05:30.466711+0000 mon.a (mon.0) 592 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:05:30.860 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:05:30 smithi089 bash[82202]: cluster 2024-07-15T07:05:28.902132+0000 mgr.y (mgr.25244) 343 : cluster [DBG] pgmap v290: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:05:30.860 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:05:30 smithi089 bash[82202]: audit 2024-07-15T07:05:29.542590+0000 mon.a (mon.0) 589 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:05:30.861 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:05:30 smithi089 bash[82202]: audit 2024-07-15T07:05:29.547117+0000 mon.a (mon.0) 590 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:05:30.861 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:05:30 smithi089 bash[82202]: audit 2024-07-15T07:05:30.456514+0000 mon.a (mon.0) 591 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:05:30.861 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:05:30 smithi089 bash[82202]: audit 2024-07-15T07:05:30.466711+0000 mon.a (mon.0) 592 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:05:31.760 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:05:31 smithi089 bash[82202]: audit 2024-07-15T07:05:30.901635+0000 mon.a (mon.0) 593 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T07:05:31.760 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:05:31 smithi089 bash[82202]: audit 2024-07-15T07:05:30.902501+0000 mon.a (mon.0) 594 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T07:05:31.760 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:05:31 smithi089 bash[82202]: audit 2024-07-15T07:05:30.910471+0000 mon.a (mon.0) 595 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:05:31.760 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:05:31 smithi089 bash[79810]: audit 2024-07-15T07:05:30.901635+0000 mon.a (mon.0) 593 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T07:05:31.760 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:05:31 smithi089 bash[79810]: audit 2024-07-15T07:05:30.902501+0000 mon.a (mon.0) 594 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T07:05:31.761 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:05:31 smithi089 bash[79810]: audit 2024-07-15T07:05:30.910471+0000 mon.a (mon.0) 595 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:05:31.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:05:31 smithi134 bash[67624]: audit 2024-07-15T07:05:30.901635+0000 mon.a (mon.0) 593 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T07:05:31.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:05:31 smithi134 bash[67624]: audit 2024-07-15T07:05:30.902501+0000 mon.a (mon.0) 594 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T07:05:31.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:05:31 smithi134 bash[67624]: audit 2024-07-15T07:05:30.910471+0000 mon.a (mon.0) 595 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:05:32.621 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:05:32 smithi089 bash[79810]: cluster 2024-07-15T07:05:30.902584+0000 mgr.y (mgr.25244) 344 : cluster [DBG] pgmap v291: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:05:32.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:05:32 smithi089 bash[82202]: cluster 2024-07-15T07:05:30.902584+0000 mgr.y (mgr.25244) 344 : cluster [DBG] pgmap v291: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:05:32.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:05:32 smithi134 bash[67624]: cluster 2024-07-15T07:05:30.902584+0000 mgr.y (mgr.25244) 344 : cluster [DBG] pgmap v291: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:05:32.806 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:05:32 smithi134 bash[60729]: ts=2024-07-15T07:05:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T07:05:34.381 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 07:05:34 smithi089 bash[72894]: ::ffff:172.21.15.134 - - [15/Jul/2024:07:05:34] "GET /metrics HTTP/1.1" 200 33979 "" "Prometheus/2.43.0" 2024-07-15T07:05:34.805 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:05:34 smithi134 bash[67624]: cluster 2024-07-15T07:05:32.903363+0000 mgr.y (mgr.25244) 345 : cluster [DBG] pgmap v292: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:05:34.915 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:05:34 smithi089 bash[82202]: cluster 2024-07-15T07:05:32.903363+0000 mgr.y (mgr.25244) 345 : cluster [DBG] pgmap v292: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:05:34.915 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:05:34 smithi089 bash[79810]: cluster 2024-07-15T07:05:32.903363+0000 mgr.y (mgr.25244) 345 : cluster [DBG] pgmap v292: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:05:35.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:05:35 smithi089 bash[79810]: audit 2024-07-15T07:05:34.795905+0000 mon.a (mon.0) 596 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:05:35.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:05:35 smithi089 bash[82202]: audit 2024-07-15T07:05:34.795905+0000 mon.a (mon.0) 596 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:05:36.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:05:35 smithi134 bash[67624]: audit 2024-07-15T07:05:34.795905+0000 mon.a (mon.0) 596 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:05:36.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:05:36 smithi089 bash[79810]: cluster 2024-07-15T07:05:34.904141+0000 mgr.y (mgr.25244) 346 : cluster [DBG] pgmap v293: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:05:36.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:05:36 smithi089 bash[82202]: cluster 2024-07-15T07:05:34.904141+0000 mgr.y (mgr.25244) 346 : cluster [DBG] pgmap v293: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:05:37.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:05:36 smithi134 bash[67624]: cluster 2024-07-15T07:05:34.904141+0000 mgr.y (mgr.25244) 346 : cluster [DBG] pgmap v293: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:05:38.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:05:38 smithi089 bash[79810]: cluster 2024-07-15T07:05:36.905236+0000 mgr.y (mgr.25244) 347 : cluster [DBG] pgmap v294: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:05:38.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:05:38 smithi089 bash[82202]: cluster 2024-07-15T07:05:36.905236+0000 mgr.y (mgr.25244) 347 : cluster [DBG] pgmap v294: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:05:39.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:05:38 smithi134 bash[67624]: cluster 2024-07-15T07:05:36.905236+0000 mgr.y (mgr.25244) 347 : cluster [DBG] pgmap v294: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:05:39.487 INFO:teuthology.orchestra.run.smithi089.stdout:438086664216 2024-07-15T07:05:39.973 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:05:39 smithi089 bash[82202]: audit 2024-07-15T07:05:39.490175+0000 mon.c (mon.2) 17 : audit [DBG] from='client.? 172.21.15.89:0/499257868' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-07-15T07:05:39.973 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:05:39 smithi089 bash[79810]: audit 2024-07-15T07:05:39.490175+0000 mon.c (mon.2) 17 : audit [DBG] from='client.? 172.21.15.89:0/499257868' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-07-15T07:05:40.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:05:39 smithi134 bash[67624]: audit 2024-07-15T07:05:39.490175+0000 mon.c (mon.2) 17 : audit [DBG] from='client.? 172.21.15.89:0/499257868' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-07-15T07:05:40.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:05:40 smithi089 bash[79810]: cluster 2024-07-15T07:05:38.905758+0000 mgr.y (mgr.25244) 348 : cluster [DBG] pgmap v295: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:05:40.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:05:40 smithi089 bash[82202]: cluster 2024-07-15T07:05:38.905758+0000 mgr.y (mgr.25244) 348 : cluster [DBG] pgmap v295: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:05:41.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:05:40 smithi134 bash[67624]: cluster 2024-07-15T07:05:38.905758+0000 mgr.y (mgr.25244) 348 : cluster [DBG] pgmap v295: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:05:41.704 INFO:teuthology.orchestra.run.smithi089.stdout:412316860447 2024-07-15T07:05:42.047 INFO:teuthology.orchestra.run.smithi089.stdout:472446402579 2024-07-15T07:05:42.572 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 07:05:42 smithi134 bash[59462]: logger=sqlstore.transactions t=2024-07-15T07:05:42.444685909Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-07-15T07:05:42.572 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:05:42 smithi134 bash[60729]: ts=2024-07-15T07:05:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T07:05:42.808 INFO:tasks.cephadm.ceph_manager.ceph:need seq 438086664212 got 438086664216 for osd.3 2024-07-15T07:05:42.808 DEBUG:teuthology.parallel:result is None 2024-07-15T07:05:42.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:05:42 smithi089 bash[79810]: cluster 2024-07-15T07:05:40.906691+0000 mgr.y (mgr.25244) 349 : cluster [DBG] pgmap v296: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:05:42.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:05:42 smithi089 bash[79810]: audit 2024-07-15T07:05:41.707585+0000 mon.a (mon.0) 597 : audit [DBG] from='client.? 172.21.15.89:0/2405270874' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-07-15T07:05:42.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:05:42 smithi089 bash[79810]: audit 2024-07-15T07:05:42.049922+0000 mon.c (mon.2) 18 : audit [DBG] from='client.? 172.21.15.89:0/3843447697' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-07-15T07:05:42.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:05:42 smithi089 bash[82202]: cluster 2024-07-15T07:05:40.906691+0000 mgr.y (mgr.25244) 349 : cluster [DBG] pgmap v296: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:05:42.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:05:42 smithi089 bash[82202]: audit 2024-07-15T07:05:41.707585+0000 mon.a (mon.0) 597 : audit [DBG] from='client.? 172.21.15.89:0/2405270874' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-07-15T07:05:42.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:05:42 smithi089 bash[82202]: audit 2024-07-15T07:05:42.049922+0000 mon.c (mon.2) 18 : audit [DBG] from='client.? 172.21.15.89:0/3843447697' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-07-15T07:05:43.056 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 07:05:42 smithi134 bash[59462]: logger=cleanup t=2024-07-15T07:05:42.576463965Z level=info msg="Completed cleanup jobs" duration=143.393817ms 2024-07-15T07:05:43.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:05:42 smithi134 bash[67624]: cluster 2024-07-15T07:05:40.906691+0000 mgr.y (mgr.25244) 349 : cluster [DBG] pgmap v296: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:05:43.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:05:42 smithi134 bash[67624]: audit 2024-07-15T07:05:41.707585+0000 mon.a (mon.0) 597 : audit [DBG] from='client.? 172.21.15.89:0/2405270874' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-07-15T07:05:43.057 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:05:42 smithi134 bash[67624]: audit 2024-07-15T07:05:42.049922+0000 mon.c (mon.2) 18 : audit [DBG] from='client.? 172.21.15.89:0/3843447697' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-07-15T07:05:43.693 INFO:teuthology.orchestra.run.smithi089.stdout:356482285623 2024-07-15T07:05:44.073 INFO:tasks.cephadm.ceph_manager.ceph:need seq 412316860442 got 412316860447 for osd.2 2024-07-15T07:05:44.074 DEBUG:teuthology.parallel:result is None 2024-07-15T07:05:44.372 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 07:05:44 smithi089 bash[72894]: ::ffff:172.21.15.134 - - [15/Jul/2024:07:05:44] "GET /metrics HTTP/1.1" 200 33980 "" "Prometheus/2.43.0" 2024-07-15T07:05:44.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:05:44 smithi089 bash[79810]: cluster 2024-07-15T07:05:42.907221+0000 mgr.y (mgr.25244) 350 : cluster [DBG] pgmap v297: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:05:44.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:05:44 smithi089 bash[79810]: audit 2024-07-15T07:05:43.696829+0000 mon.a (mon.0) 598 : audit [DBG] from='client.? 172.21.15.89:0/2057496000' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-07-15T07:05:44.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:05:44 smithi089 bash[82202]: cluster 2024-07-15T07:05:42.907221+0000 mgr.y (mgr.25244) 350 : cluster [DBG] pgmap v297: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:05:44.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:05:44 smithi089 bash[82202]: audit 2024-07-15T07:05:43.696829+0000 mon.a (mon.0) 598 : audit [DBG] from='client.? 172.21.15.89:0/2057496000' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-07-15T07:05:44.947 INFO:tasks.cephadm.ceph_manager.ceph:need seq 472446402575 got 472446402579 for osd.4 2024-07-15T07:05:44.947 DEBUG:teuthology.parallel:result is None 2024-07-15T07:05:45.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:05:44 smithi134 bash[67624]: cluster 2024-07-15T07:05:42.907221+0000 mgr.y (mgr.25244) 350 : cluster [DBG] pgmap v297: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:05:45.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:05:44 smithi134 bash[67624]: audit 2024-07-15T07:05:43.696829+0000 mon.a (mon.0) 598 : audit [DBG] from='client.? 172.21.15.89:0/2057496000' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-07-15T07:05:45.793 INFO:tasks.cephadm.ceph_manager.ceph:need seq 356482285618 got 356482285623 for osd.0 2024-07-15T07:05:45.793 DEBUG:teuthology.parallel:result is None 2024-07-15T07:05:45.986 INFO:teuthology.orchestra.run.smithi089.stdout:197568495858 2024-07-15T07:05:46.113 INFO:teuthology.orchestra.run.smithi089.stdout:231928234216 2024-07-15T07:05:46.182 INFO:teuthology.orchestra.run.smithi089.stdout:163208757500 2024-07-15T07:05:46.246 INFO:teuthology.orchestra.run.smithi089.stdout:377957122097 2024-07-15T07:05:46.904 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:05:46 smithi089 bash[79810]: cluster 2024-07-15T07:05:44.908068+0000 mgr.y (mgr.25244) 351 : cluster [DBG] pgmap v298: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:05:46.904 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:05:46 smithi089 bash[79810]: audit 2024-07-15T07:05:45.987665+0000 mon.a (mon.0) 599 : audit [DBG] from='client.? 172.21.15.89:0/1453812301' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-07-15T07:05:46.904 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:05:46 smithi089 bash[79810]: audit 2024-07-15T07:05:46.116166+0000 mon.c (mon.2) 19 : audit [DBG] from='client.? 172.21.15.89:0/46039946' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-07-15T07:05:46.904 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:05:46 smithi089 bash[79810]: audit 2024-07-15T07:05:46.185417+0000 mon.c (mon.2) 20 : audit [DBG] from='client.? 172.21.15.89:0/1032747102' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-07-15T07:05:46.904 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:05:46 smithi089 bash[79810]: audit 2024-07-15T07:05:46.247549+0000 mon.a (mon.0) 600 : audit [DBG] from='client.? 172.21.15.89:0/2158179388' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-07-15T07:05:46.905 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:05:46 smithi089 bash[82202]: cluster 2024-07-15T07:05:44.908068+0000 mgr.y (mgr.25244) 351 : cluster [DBG] pgmap v298: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:05:46.905 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:05:46 smithi089 bash[82202]: audit 2024-07-15T07:05:45.987665+0000 mon.a (mon.0) 599 : audit [DBG] from='client.? 172.21.15.89:0/1453812301' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-07-15T07:05:46.905 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:05:46 smithi089 bash[82202]: audit 2024-07-15T07:05:46.116166+0000 mon.c (mon.2) 19 : audit [DBG] from='client.? 172.21.15.89:0/46039946' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-07-15T07:05:46.905 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:05:46 smithi089 bash[82202]: audit 2024-07-15T07:05:46.185417+0000 mon.c (mon.2) 20 : audit [DBG] from='client.? 172.21.15.89:0/1032747102' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-07-15T07:05:46.905 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:05:46 smithi089 bash[82202]: audit 2024-07-15T07:05:46.247549+0000 mon.a (mon.0) 600 : audit [DBG] from='client.? 172.21.15.89:0/2158179388' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-07-15T07:05:47.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:05:46 smithi134 bash[67624]: cluster 2024-07-15T07:05:44.908068+0000 mgr.y (mgr.25244) 351 : cluster [DBG] pgmap v298: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:05:47.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:05:46 smithi134 bash[67624]: audit 2024-07-15T07:05:45.987665+0000 mon.a (mon.0) 599 : audit [DBG] from='client.? 172.21.15.89:0/1453812301' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-07-15T07:05:47.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:05:46 smithi134 bash[67624]: audit 2024-07-15T07:05:46.116166+0000 mon.c (mon.2) 19 : audit [DBG] from='client.? 172.21.15.89:0/46039946' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-07-15T07:05:47.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:05:46 smithi134 bash[67624]: audit 2024-07-15T07:05:46.185417+0000 mon.c (mon.2) 20 : audit [DBG] from='client.? 172.21.15.89:0/1032747102' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-07-15T07:05:47.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:05:46 smithi134 bash[67624]: audit 2024-07-15T07:05:46.247549+0000 mon.a (mon.0) 600 : audit [DBG] from='client.? 172.21.15.89:0/2158179388' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-07-15T07:05:47.623 INFO:tasks.cephadm.ceph_manager.ceph:need seq 197568495854 got 197568495858 for osd.6 2024-07-15T07:05:47.623 DEBUG:teuthology.parallel:result is None 2024-07-15T07:05:47.979 INFO:tasks.cephadm.ceph_manager.ceph:need seq 231928234212 got 231928234216 for osd.7 2024-07-15T07:05:47.979 DEBUG:teuthology.parallel:result is None 2024-07-15T07:05:48.307 INFO:tasks.cephadm.ceph_manager.ceph:need seq 377957122092 got 377957122097 for osd.1 2024-07-15T07:05:48.307 DEBUG:teuthology.parallel:result is None 2024-07-15T07:05:48.527 INFO:tasks.cephadm.ceph_manager.ceph:need seq 163208757495 got 163208757500 for osd.5 2024-07-15T07:05:48.527 DEBUG:teuthology.parallel:result is None 2024-07-15T07:05:48.527 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-07-15T07:05:48.527 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph pg dump --format=json 2024-07-15T07:05:48.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:05:48 smithi089 bash[79810]: cluster 2024-07-15T07:05:46.909059+0000 mgr.y (mgr.25244) 352 : cluster [DBG] pgmap v299: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:05:48.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:05:48 smithi089 bash[82202]: cluster 2024-07-15T07:05:46.909059+0000 mgr.y (mgr.25244) 352 : cluster [DBG] pgmap v299: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:05:49.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:05:48 smithi134 bash[67624]: cluster 2024-07-15T07:05:46.909059+0000 mgr.y (mgr.25244) 352 : cluster [DBG] pgmap v299: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:05:50.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:05:50 smithi089 bash[79810]: cluster 2024-07-15T07:05:48.909774+0000 mgr.y (mgr.25244) 353 : cluster [DBG] pgmap v300: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:05:50.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:05:50 smithi089 bash[79810]: audit 2024-07-15T07:05:49.796845+0000 mon.a (mon.0) 601 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:05:50.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:05:50 smithi089 bash[82202]: cluster 2024-07-15T07:05:48.909774+0000 mgr.y (mgr.25244) 353 : cluster [DBG] pgmap v300: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:05:50.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:05:50 smithi089 bash[82202]: audit 2024-07-15T07:05:49.796845+0000 mon.a (mon.0) 601 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:05:51.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:05:50 smithi134 bash[67624]: cluster 2024-07-15T07:05:48.909774+0000 mgr.y (mgr.25244) 353 : cluster [DBG] pgmap v300: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:05:51.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:05:50 smithi134 bash[67624]: audit 2024-07-15T07:05:49.796845+0000 mon.a (mon.0) 601 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:05:52.604 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:05:52 smithi134 bash[60729]: ts=2024-07-15T07:05:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T07:05:52.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:05:52 smithi089 bash[79810]: cluster 2024-07-15T07:05:50.910832+0000 mgr.y (mgr.25244) 354 : cluster [DBG] pgmap v301: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:05:52.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:05:52 smithi089 bash[82202]: cluster 2024-07-15T07:05:50.910832+0000 mgr.y (mgr.25244) 354 : cluster [DBG] pgmap v301: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:05:53.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:05:52 smithi134 bash[67624]: cluster 2024-07-15T07:05:50.910832+0000 mgr.y (mgr.25244) 354 : cluster [DBG] pgmap v301: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:05:53.313 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T07:05:54.607 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 07:05:54 smithi089 bash[72894]: ::ffff:172.21.15.134 - - [15/Jul/2024:07:05:54] "GET /metrics HTTP/1.1" 200 33974 "" "Prometheus/2.43.0" 2024-07-15T07:05:54.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:05:54 smithi089 bash[79810]: cluster 2024-07-15T07:05:52.911551+0000 mgr.y (mgr.25244) 355 : cluster [DBG] pgmap v302: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:05:54.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:05:54 smithi089 bash[82202]: cluster 2024-07-15T07:05:52.911551+0000 mgr.y (mgr.25244) 355 : cluster [DBG] pgmap v302: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:05:55.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:05:54 smithi134 bash[67624]: cluster 2024-07-15T07:05:52.911551+0000 mgr.y (mgr.25244) 355 : cluster [DBG] pgmap v302: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:05:56.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:05:56 smithi089 bash[79810]: cluster 2024-07-15T07:05:54.912240+0000 mgr.y (mgr.25244) 356 : cluster [DBG] pgmap v303: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:05:56.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:05:56 smithi089 bash[82202]: cluster 2024-07-15T07:05:54.912240+0000 mgr.y (mgr.25244) 356 : cluster [DBG] pgmap v303: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:05:57.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:05:56 smithi134 bash[67624]: cluster 2024-07-15T07:05:54.912240+0000 mgr.y (mgr.25244) 356 : cluster [DBG] pgmap v303: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:05:57.059 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-07-15T07:05:57.060 INFO:teuthology.orchestra.run.smithi089.stderr:dumped all 2024-07-15T07:05:57.759 INFO:teuthology.orchestra.run.smithi089.stdout:{"pg_ready":true,"pg_map":{"version":304,"stamp":"2024-07-15T07:05:56.912547+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595910,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":537,"num_read_kb":963,"num_write":300,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":1771244,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":222,"ondisk_log_size":222,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1078536,"kb_used_data":11928,"kb_used_omap":109,"kb_used_meta":1066386,"kb_avail":748882680,"statfs":{"total":767960285184,"available":766855864320,"internally_reserved":0,"allocated":12214272,"data_stored":7277868,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":111767,"internal_metadata":1091980137},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.005104"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":47,"reported_epoch":114,"state":"active+clean","last_fresh":"2024-07-15T07:05:07.657645+0000","last_change":"2024-07-15T07:04:17.894309+0000","last_active":"2024-07-15T07:05:07.657645+0000","last_peered":"2024-07-15T07:05:07.657645+0000","last_clean":"2024-07-15T07:05:07.657645+0000","last_became_active":"2024-07-15T07:04:17.893694+0000","last_became_peered":"2024-07-15T07:04:17.893694+0000","last_unstale":"2024-07-15T07:05:07.657645+0000","last_undegraded":"2024-07-15T07:05:07.657645+0000","last_fullsized":"2024-07-15T07:05:07.657645+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T17:37:31.349700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":95,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-07-15T07:05:05.975211+0000","last_change":"2024-07-15T07:04:16.928586+0000","last_active":"2024-07-15T07:05:05.975211+0000","last_peered":"2024-07-15T07:05:05.975211+0000","last_clean":"2024-07-15T07:05:05.975211+0000","last_became_active":"2024-07-15T07:04:16.927823+0000","last_became_peered":"2024-07-15T07:04:16.927823+0000","last_unstale":"2024-07-15T07:05:05.975211+0000","last_undegraded":"2024-07-15T07:05:05.975211+0000","last_fullsized":"2024-07-15T07:05:05.975211+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T15:07:07.863429+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":7,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":74,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-07-15T07:05:05.972262+0000","last_change":"2024-07-15T07:03:48.026084+0000","last_active":"2024-07-15T07:05:05.972262+0000","last_peered":"2024-07-15T07:05:05.972262+0000","last_clean":"2024-07-15T07:05:05.972262+0000","last_became_active":"2024-07-15T07:03:48.024868+0000","last_became_peered":"2024-07-15T07:03:48.024868+0000","last_unstale":"2024-07-15T07:05:05.972262+0000","last_undegraded":"2024-07-15T07:05:05.972262+0000","last_fullsized":"2024-07-15T07:05:05.972262+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T16:20:53.777311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":67,"reported_epoch":114,"state":"active+clean","last_fresh":"2024-07-15T07:05:07.668195+0000","last_change":"2024-07-15T07:03:48.041561+0000","last_active":"2024-07-15T07:05:07.668195+0000","last_peered":"2024-07-15T07:05:07.668195+0000","last_clean":"2024-07-15T07:05:07.668195+0000","last_became_active":"2024-07-15T07:03:48.040745+0000","last_became_peered":"2024-07-15T07:03:48.040745+0000","last_unstale":"2024-07-15T07:05:07.668195+0000","last_undegraded":"2024-07-15T07:05:07.668195+0000","last_fullsized":"2024-07-15T07:05:07.668195+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T15:33:44.273459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":66,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-07-15T07:05:06.450803+0000","last_change":"2024-07-15T07:03:48.048307+0000","last_active":"2024-07-15T07:05:06.450803+0000","last_peered":"2024-07-15T07:05:06.450803+0000","last_clean":"2024-07-15T07:05:06.450803+0000","last_became_active":"2024-07-15T07:03:48.047249+0000","last_became_peered":"2024-07-15T07:03:48.047249+0000","last_unstale":"2024-07-15T07:05:06.450803+0000","last_undegraded":"2024-07-15T07:05:06.450803+0000","last_fullsized":"2024-07-15T07:05:06.450803+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T08:21:37.555562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":74,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-07-15T07:05:05.974284+0000","last_change":"2024-07-15T07:04:16.931237+0000","last_active":"2024-07-15T07:05:05.974284+0000","last_peered":"2024-07-15T07:05:05.974284+0000","last_clean":"2024-07-15T07:05:05.974284+0000","last_became_active":"2024-07-15T07:04:16.930149+0000","last_became_peered":"2024-07-15T07:04:16.930149+0000","last_unstale":"2024-07-15T07:05:05.974284+0000","last_undegraded":"2024-07-15T07:05:05.974284+0000","last_fullsized":"2024-07-15T07:05:05.974284+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T13:02:49.562178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":27,"reported_epoch":114,"state":"active+clean","last_fresh":"2024-07-15T07:05:07.658042+0000","last_change":"2024-07-15T07:03:18.567697+0000","last_active":"2024-07-15T07:05:07.658042+0000","last_peered":"2024-07-15T07:05:07.658042+0000","last_clean":"2024-07-15T07:05:07.658042+0000","last_became_active":"2024-07-15T07:03:18.566294+0000","last_became_peered":"2024-07-15T07:03:18.566294+0000","last_unstale":"2024-07-15T07:05:07.658042+0000","last_undegraded":"2024-07-15T07:05:07.658042+0000","last_fullsized":"2024-07-15T07:05:07.658042+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T15:04:49.724233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":86,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-07-15T07:05:05.964210+0000","last_change":"2024-07-15T07:01:15.785260+0000","last_active":"2024-07-15T07:05:05.964210+0000","last_peered":"2024-07-15T07:05:05.964210+0000","last_clean":"2024-07-15T07:05:05.964210+0000","last_became_active":"2024-07-15T07:01:15.783889+0000","last_became_peered":"2024-07-15T07:01:15.783889+0000","last_unstale":"2024-07-15T07:05:05.964210+0000","last_undegraded":"2024-07-15T07:05:05.964210+0000","last_fullsized":"2024-07-15T07:05:05.964210+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T17:22:29.018297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":12,"num_read_kb":12,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":93,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-07-15T07:05:05.976353+0000","last_change":"2024-07-15T07:04:16.931104+0000","last_active":"2024-07-15T07:05:05.976353+0000","last_peered":"2024-07-15T07:05:05.976353+0000","last_clean":"2024-07-15T07:05:05.976353+0000","last_became_active":"2024-07-15T07:04:16.930135+0000","last_became_peered":"2024-07-15T07:04:16.930135+0000","last_unstale":"2024-07-15T07:05:05.976353+0000","last_undegraded":"2024-07-15T07:05:05.976353+0000","last_fullsized":"2024-07-15T07:05:05.976353+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T12:33:45.251039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":57,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-07-15T07:05:05.970906+0000","last_change":"2024-07-15T06:50:29.845162+0000","last_active":"2024-07-15T07:05:05.970906+0000","last_peered":"2024-07-15T07:05:05.970906+0000","last_clean":"2024-07-15T07:05:05.970906+0000","last_became_active":"2024-07-15T06:50:29.844204+0000","last_became_peered":"2024-07-15T06:50:29.844204+0000","last_unstale":"2024-07-15T07:05:05.970906+0000","last_undegraded":"2024-07-15T07:05:05.970906+0000","last_fullsized":"2024-07-15T07:05:05.970906+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T16:56:12.605992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":94,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-07-15T07:05:05.976174+0000","last_change":"2024-07-15T07:04:16.924775+0000","last_active":"2024-07-15T07:05:05.976174+0000","last_peered":"2024-07-15T07:05:05.976174+0000","last_clean":"2024-07-15T07:05:05.976174+0000","last_became_active":"2024-07-15T07:04:16.923784+0000","last_became_peered":"2024-07-15T07:04:16.923784+0000","last_unstale":"2024-07-15T07:05:05.976174+0000","last_undegraded":"2024-07-15T07:05:05.976174+0000","last_fullsized":"2024-07-15T07:05:05.976174+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T17:51:23.601587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":95,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-07-15T07:05:05.967573+0000","last_change":"2024-07-15T07:04:16.915477+0000","last_active":"2024-07-15T07:05:05.967573+0000","last_peered":"2024-07-15T07:05:05.967573+0000","last_clean":"2024-07-15T07:05:05.967573+0000","last_became_active":"2024-07-15T07:04:16.913839+0000","last_became_peered":"2024-07-15T07:04:16.913839+0000","last_unstale":"2024-07-15T07:05:05.967573+0000","last_undegraded":"2024-07-15T07:05:05.967573+0000","last_fullsized":"2024-07-15T07:05:05.967573+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T07:02:16.971182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":14,"reported_epoch":114,"state":"active+clean","last_fresh":"2024-07-15T07:05:06.988129+0000","last_change":"2024-07-15T07:05:06.988129+0000","last_active":"2024-07-15T07:05:06.988129+0000","last_peered":"2024-07-15T07:05:06.988129+0000","last_clean":"2024-07-15T07:05:06.988129+0000","last_became_active":"2024-07-15T07:05:06.982223+0000","last_became_peered":"2024-07-15T07:05:06.982223+0000","last_unstale":"2024-07-15T07:05:06.988129+0000","last_undegraded":"2024-07-15T07:05:06.988129+0000","last_fullsized":"2024-07-15T07:05:06.988129+0000","mapping_epoch":113,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T08:19:38.225384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":83,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-07-15T07:05:05.964238+0000","last_change":"2024-07-15T07:03:17.337106+0000","last_active":"2024-07-15T07:05:05.964238+0000","last_peered":"2024-07-15T07:05:05.964238+0000","last_clean":"2024-07-15T07:05:05.964238+0000","last_became_active":"2024-07-15T07:03:17.336185+0000","last_became_peered":"2024-07-15T07:03:17.336185+0000","last_unstale":"2024-07-15T07:05:05.964238+0000","last_undegraded":"2024-07-15T07:05:05.964238+0000","last_fullsized":"2024-07-15T07:05:05.964238+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T12:00:14.906125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":75,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-07-15T07:05:05.971882+0000","last_change":"2024-07-15T07:03:48.036322+0000","last_active":"2024-07-15T07:05:05.971882+0000","last_peered":"2024-07-15T07:05:05.971882+0000","last_clean":"2024-07-15T07:05:05.971882+0000","last_became_active":"2024-07-15T07:03:48.035590+0000","last_became_peered":"2024-07-15T07:03:48.035590+0000","last_unstale":"2024-07-15T07:05:05.971882+0000","last_undegraded":"2024-07-15T07:05:05.971882+0000","last_fullsized":"2024-07-15T07:05:05.971882+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T07:31:38.707447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":82,"reported_epoch":114,"state":"active+clean","last_fresh":"2024-07-15T07:05:07.668548+0000","last_change":"2024-07-15T07:04:16.914320+0000","last_active":"2024-07-15T07:05:07.668548+0000","last_peered":"2024-07-15T07:05:07.668548+0000","last_clean":"2024-07-15T07:05:07.668548+0000","last_became_active":"2024-07-15T07:04:16.913426+0000","last_became_peered":"2024-07-15T07:04:16.913426+0000","last_unstale":"2024-07-15T07:05:07.668548+0000","last_undegraded":"2024-07-15T07:05:07.668548+0000","last_fullsized":"2024-07-15T07:05:07.668548+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T10:33:29.340996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"68'3","reported_seq":88,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-07-15T07:05:05.964023+0000","last_change":"2024-07-15T07:04:16.914557+0000","last_active":"2024-07-15T07:05:05.964023+0000","last_peered":"2024-07-15T07:05:05.964023+0000","last_clean":"2024-07-15T07:05:05.964023+0000","last_became_active":"2024-07-15T07:04:16.913557+0000","last_became_peered":"2024-07-15T07:04:16.913557+0000","last_unstale":"2024-07-15T07:05:05.964023+0000","last_undegraded":"2024-07-15T07:05:05.964023+0000","last_fullsized":"2024-07-15T07:05:05.964023+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T15:01:08.453349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":13,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-07-15T07:05:05.965499+0000","last_change":"2024-07-15T07:04:17.897049+0000","last_active":"2024-07-15T07:05:05.965499+0000","last_peered":"2024-07-15T07:05:05.965499+0000","last_clean":"2024-07-15T07:05:05.965499+0000","last_became_active":"2024-07-15T07:04:17.895834+0000","last_became_peered":"2024-07-15T07:04:17.895834+0000","last_unstale":"2024-07-15T07:05:05.965499+0000","last_undegraded":"2024-07-15T07:05:05.965499+0000","last_fullsized":"2024-07-15T07:05:05.965499+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T15:15:26.611263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":91,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-07-15T07:05:05.964527+0000","last_change":"2024-07-15T07:04:16.916338+0000","last_active":"2024-07-15T07:05:05.964527+0000","last_peered":"2024-07-15T07:05:05.964527+0000","last_clean":"2024-07-15T07:05:05.964527+0000","last_became_active":"2024-07-15T07:04:16.915416+0000","last_became_peered":"2024-07-15T07:04:16.915416+0000","last_unstale":"2024-07-15T07:05:05.964527+0000","last_undegraded":"2024-07-15T07:05:05.964527+0000","last_fullsized":"2024-07-15T07:05:05.964527+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T07:03:04.534165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":90,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-07-15T07:05:05.967781+0000","last_change":"2024-07-15T07:04:16.919574+0000","last_active":"2024-07-15T07:05:05.967781+0000","last_peered":"2024-07-15T07:05:05.967781+0000","last_clean":"2024-07-15T07:05:05.967781+0000","last_became_active":"2024-07-15T07:04:16.916288+0000","last_became_peered":"2024-07-15T07:04:16.916288+0000","last_unstale":"2024-07-15T07:05:05.967781+0000","last_undegraded":"2024-07-15T07:05:05.967781+0000","last_fullsized":"2024-07-15T07:05:05.967781+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T17:21:49.043384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":74,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-07-15T07:05:05.967299+0000","last_change":"2024-07-15T07:04:16.924979+0000","last_active":"2024-07-15T07:05:05.967299+0000","last_peered":"2024-07-15T07:05:05.967299+0000","last_clean":"2024-07-15T07:05:05.967299+0000","last_became_active":"2024-07-15T07:04:16.924217+0000","last_became_peered":"2024-07-15T07:04:16.924217+0000","last_unstale":"2024-07-15T07:05:05.967299+0000","last_undegraded":"2024-07-15T07:05:05.967299+0000","last_fullsized":"2024-07-15T07:05:05.967299+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T18:45:26.997132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":79,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-07-15T07:05:05.966025+0000","last_change":"2024-07-15T07:04:16.912026+0000","last_active":"2024-07-15T07:05:05.966025+0000","last_peered":"2024-07-15T07:05:05.966025+0000","last_clean":"2024-07-15T07:05:05.966025+0000","last_became_active":"2024-07-15T07:04:16.910983+0000","last_became_peered":"2024-07-15T07:04:16.910983+0000","last_unstale":"2024-07-15T07:05:05.966025+0000","last_undegraded":"2024-07-15T07:05:05.966025+0000","last_fullsized":"2024-07-15T07:05:05.966025+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T11:10:26.472987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":74,"reported_epoch":114,"state":"active+clean","last_fresh":"2024-07-15T07:05:07.669104+0000","last_change":"2024-07-15T07:03:48.037027+0000","last_active":"2024-07-15T07:05:07.669104+0000","last_peered":"2024-07-15T07:05:07.669104+0000","last_clean":"2024-07-15T07:05:07.669104+0000","last_became_active":"2024-07-15T07:03:48.035771+0000","last_became_peered":"2024-07-15T07:03:48.035771+0000","last_unstale":"2024-07-15T07:05:07.669104+0000","last_undegraded":"2024-07-15T07:05:07.669104+0000","last_fullsized":"2024-07-15T07:05:07.669104+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T08:50:35.241323+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":57,"reported_epoch":114,"state":"active+clean","last_fresh":"2024-07-15T07:05:07.658011+0000","last_change":"2024-07-15T07:04:16.915793+0000","last_active":"2024-07-15T07:05:07.658011+0000","last_peered":"2024-07-15T07:05:07.658011+0000","last_clean":"2024-07-15T07:05:07.658011+0000","last_became_active":"2024-07-15T07:04:16.914806+0000","last_became_peered":"2024-07-15T07:04:16.914806+0000","last_unstale":"2024-07-15T07:05:07.658011+0000","last_undegraded":"2024-07-15T07:05:07.658011+0000","last_fullsized":"2024-07-15T07:05:07.658011+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T16:42:59.639522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"79'161","reported_seq":454,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-07-15T07:05:05.973920+0000","last_change":"2024-07-15T07:01:15.788351+0000","last_active":"2024-07-15T07:05:05.973920+0000","last_peered":"2024-07-15T07:05:05.973920+0000","last_clean":"2024-07-15T07:05:05.973920+0000","last_became_active":"2024-07-15T07:01:15.787325+0000","last_became_peered":"2024-07-15T07:01:15.787325+0000","last_unstale":"2024-07-15T07:05:05.973920+0000","last_undegraded":"2024-07-15T07:05:05.973920+0000","last_fullsized":"2024-07-15T07:05:05.973920+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:42:21.582843+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:42:21.582843+0000","last_clean_scrub_stamp":"2024-07-15T06:42:21.582843+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T11:32:38.840763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":79,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-07-15T07:05:05.971764+0000","last_change":"2024-07-15T07:03:17.344493+0000","last_active":"2024-07-15T07:05:05.971764+0000","last_peered":"2024-07-15T07:05:05.971764+0000","last_clean":"2024-07-15T07:05:05.971764+0000","last_became_active":"2024-07-15T07:03:17.343699+0000","last_became_peered":"2024-07-15T07:03:17.343699+0000","last_unstale":"2024-07-15T07:05:05.971764+0000","last_undegraded":"2024-07-15T07:05:05.971764+0000","last_fullsized":"2024-07-15T07:05:05.971764+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_clean_scrub_stamp":"2024-07-15T06:48:27.655327+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T11:46:10.753577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":34,"reported_epoch":114,"state":"active+clean","last_fresh":"2024-07-15T07:05:07.669847+0000","last_change":"2024-07-15T07:03:48.042626+0000","last_active":"2024-07-15T07:05:07.669847+0000","last_peered":"2024-07-15T07:05:07.669847+0000","last_clean":"2024-07-15T07:05:07.669847+0000","last_became_active":"2024-07-15T07:03:48.041677+0000","last_became_peered":"2024-07-15T07:03:48.041677+0000","last_unstale":"2024-07-15T07:05:07.669847+0000","last_undegraded":"2024-07-15T07:05:07.669847+0000","last_fullsized":"2024-07-15T07:05:07.669847+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T14:30:01.080314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":73,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-07-15T07:05:05.965689+0000","last_change":"2024-07-15T07:04:16.918686+0000","last_active":"2024-07-15T07:05:05.965689+0000","last_peered":"2024-07-15T07:05:05.965689+0000","last_clean":"2024-07-15T07:05:05.965689+0000","last_became_active":"2024-07-15T07:04:16.917900+0000","last_became_peered":"2024-07-15T07:04:16.917900+0000","last_unstale":"2024-07-15T07:05:05.965689+0000","last_undegraded":"2024-07-15T07:05:05.965689+0000","last_fullsized":"2024-07-15T07:05:05.965689+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T09:00:26.561257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":95,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-07-15T07:05:05.973817+0000","last_change":"2024-07-15T07:01:48.416152+0000","last_active":"2024-07-15T07:05:05.973817+0000","last_peered":"2024-07-15T07:05:05.973817+0000","last_clean":"2024-07-15T07:05:05.973817+0000","last_became_active":"2024-07-15T07:01:48.415553+0000","last_became_peered":"2024-07-15T07:01:48.415553+0000","last_unstale":"2024-07-15T07:05:05.973817+0000","last_undegraded":"2024-07-15T07:05:05.973817+0000","last_fullsized":"2024-07-15T07:05:05.973817+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_clean_scrub_stamp":"2024-07-15T06:48:27.655327+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T10:50:00.600843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":82,"reported_epoch":114,"state":"active+clean","last_fresh":"2024-07-15T07:05:07.669135+0000","last_change":"2024-07-15T07:04:16.910324+0000","last_active":"2024-07-15T07:05:07.669135+0000","last_peered":"2024-07-15T07:05:07.669135+0000","last_clean":"2024-07-15T07:05:07.669135+0000","last_became_active":"2024-07-15T07:04:16.909527+0000","last_became_peered":"2024-07-15T07:04:16.909527+0000","last_unstale":"2024-07-15T07:05:07.669135+0000","last_undegraded":"2024-07-15T07:05:07.669135+0000","last_fullsized":"2024-07-15T07:05:07.669135+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T07:20:18.914775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":82,"reported_epoch":114,"state":"active+clean","last_fresh":"2024-07-15T07:05:07.668858+0000","last_change":"2024-07-15T07:03:17.331985+0000","last_active":"2024-07-15T07:05:07.668858+0000","last_peered":"2024-07-15T07:05:07.668858+0000","last_clean":"2024-07-15T07:05:07.668858+0000","last_became_active":"2024-07-15T07:03:17.331049+0000","last_became_peered":"2024-07-15T07:03:17.331049+0000","last_unstale":"2024-07-15T07:05:07.668858+0000","last_undegraded":"2024-07-15T07:05:07.668858+0000","last_fullsized":"2024-07-15T07:05:07.668858+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T07:27:20.649508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":94,"reported_epoch":114,"state":"active+clean","last_fresh":"2024-07-15T07:05:07.657352+0000","last_change":"2024-07-15T07:03:48.034592+0000","last_active":"2024-07-15T07:05:07.657352+0000","last_peered":"2024-07-15T07:05:07.657352+0000","last_clean":"2024-07-15T07:05:07.657352+0000","last_became_active":"2024-07-15T07:03:48.033864+0000","last_became_peered":"2024-07-15T07:03:48.033864+0000","last_unstale":"2024-07-15T07:05:07.657352+0000","last_undegraded":"2024-07-15T07:05:07.657352+0000","last_fullsized":"2024-07-15T07:05:07.657352+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_clean_scrub_stamp":"2024-07-15T06:48:27.655327+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T07:20:45.251256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":64,"reported_epoch":114,"state":"active+clean","last_fresh":"2024-07-15T07:05:07.671334+0000","last_change":"2024-07-15T07:03:17.330796+0000","last_active":"2024-07-15T07:05:07.671334+0000","last_peered":"2024-07-15T07:05:07.671334+0000","last_clean":"2024-07-15T07:05:07.671334+0000","last_became_active":"2024-07-15T07:03:17.329922+0000","last_became_peered":"2024-07-15T07:03:17.329922+0000","last_unstale":"2024-07-15T07:05:07.671334+0000","last_undegraded":"2024-07-15T07:05:07.671334+0000","last_fullsized":"2024-07-15T07:05:07.671334+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T08:39:59.883876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":82,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-07-15T07:05:06.450880+0000","last_change":"2024-07-15T07:01:48.417853+0000","last_active":"2024-07-15T07:05:06.450880+0000","last_peered":"2024-07-15T07:05:06.450880+0000","last_clean":"2024-07-15T07:05:06.450880+0000","last_became_active":"2024-07-15T07:01:48.413264+0000","last_became_peered":"2024-07-15T07:01:48.413264+0000","last_unstale":"2024-07-15T07:05:06.450880+0000","last_undegraded":"2024-07-15T07:05:06.450880+0000","last_fullsized":"2024-07-15T07:05:06.450880+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T07:11:58.680660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":79,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-07-15T07:05:05.971399+0000","last_change":"2024-07-15T07:01:48.412889+0000","last_active":"2024-07-15T07:05:05.971399+0000","last_peered":"2024-07-15T07:05:05.971399+0000","last_clean":"2024-07-15T07:05:05.971399+0000","last_became_active":"2024-07-15T07:01:48.411325+0000","last_became_peered":"2024-07-15T07:01:48.411325+0000","last_unstale":"2024-07-15T07:05:05.971399+0000","last_undegraded":"2024-07-15T07:05:05.971399+0000","last_fullsized":"2024-07-15T07:05:05.971399+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_clean_scrub_stamp":"2024-07-15T06:48:27.655327+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T08:05:07.677914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":61,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-07-15T07:05:05.965280+0000","last_change":"2024-07-15T07:04:16.908785+0000","last_active":"2024-07-15T07:05:05.965280+0000","last_peered":"2024-07-15T07:05:05.965280+0000","last_clean":"2024-07-15T07:05:05.965280+0000","last_became_active":"2024-07-15T07:04:16.908008+0000","last_became_peered":"2024-07-15T07:04:16.908008+0000","last_unstale":"2024-07-15T07:05:05.965280+0000","last_undegraded":"2024-07-15T07:05:05.965280+0000","last_fullsized":"2024-07-15T07:05:05.965280+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T13:32:11.951412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":62,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-07-15T07:05:05.971480+0000","last_change":"2024-07-15T07:03:48.024815+0000","last_active":"2024-07-15T07:05:05.971480+0000","last_peered":"2024-07-15T07:05:05.971480+0000","last_clean":"2024-07-15T07:05:05.971480+0000","last_became_active":"2024-07-15T07:03:48.023717+0000","last_became_peered":"2024-07-15T07:03:48.023717+0000","last_unstale":"2024-07-15T07:05:05.971480+0000","last_undegraded":"2024-07-15T07:05:05.971480+0000","last_fullsized":"2024-07-15T07:05:05.971480+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T17:02:25.635190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":126,"reported_epoch":114,"state":"active+clean","last_fresh":"2024-07-15T07:05:07.671975+0000","last_change":"2024-07-15T07:05:07.671975+0000","last_active":"2024-07-15T07:05:07.671975+0000","last_peered":"2024-07-15T07:05:07.671975+0000","last_clean":"2024-07-15T07:05:07.671975+0000","last_became_active":"2024-07-15T07:05:07.671388+0000","last_became_peered":"2024-07-15T07:05:07.671388+0000","last_unstale":"2024-07-15T07:05:07.671975+0000","last_undegraded":"2024-07-15T07:05:07.671975+0000","last_fullsized":"2024-07-15T07:05:07.671975+0000","mapping_epoch":113,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T07:23:01.512927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":88,"reported_epoch":114,"state":"active+clean","last_fresh":"2024-07-15T07:05:07.672339+0000","last_change":"2024-07-15T07:01:48.419357+0000","last_active":"2024-07-15T07:05:07.672339+0000","last_peered":"2024-07-15T07:05:07.672339+0000","last_clean":"2024-07-15T07:05:07.672339+0000","last_became_active":"2024-07-15T07:01:48.417961+0000","last_became_peered":"2024-07-15T07:01:48.417961+0000","last_unstale":"2024-07-15T07:05:07.672339+0000","last_undegraded":"2024-07-15T07:05:07.672339+0000","last_fullsized":"2024-07-15T07:05:07.672339+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_clean_scrub_stamp":"2024-07-15T06:48:27.655327+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T16:29:08.322733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":64,"reported_epoch":114,"state":"active+clean","last_fresh":"2024-07-15T07:05:07.672250+0000","last_change":"2024-07-15T07:04:16.905514+0000","last_active":"2024-07-15T07:05:07.672250+0000","last_peered":"2024-07-15T07:05:07.672250+0000","last_clean":"2024-07-15T07:05:07.672250+0000","last_became_active":"2024-07-15T07:04:16.904272+0000","last_became_peered":"2024-07-15T07:04:16.904272+0000","last_unstale":"2024-07-15T07:05:07.672250+0000","last_undegraded":"2024-07-15T07:05:07.672250+0000","last_fullsized":"2024-07-15T07:05:07.672250+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T16:34:30.504887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":64,"reported_epoch":114,"state":"active+clean","last_fresh":"2024-07-15T07:05:07.672364+0000","last_change":"2024-07-15T07:03:17.339788+0000","last_active":"2024-07-15T07:05:07.672364+0000","last_peered":"2024-07-15T07:05:07.672364+0000","last_clean":"2024-07-15T07:05:07.672364+0000","last_became_active":"2024-07-15T07:03:17.338744+0000","last_became_peered":"2024-07-15T07:03:17.338744+0000","last_unstale":"2024-07-15T07:05:07.672364+0000","last_undegraded":"2024-07-15T07:05:07.672364+0000","last_fullsized":"2024-07-15T07:05:07.672364+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T11:12:21.485090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":95,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-07-15T07:05:05.976848+0000","last_change":"2024-07-15T07:04:16.929970+0000","last_active":"2024-07-15T07:05:05.976848+0000","last_peered":"2024-07-15T07:05:05.976848+0000","last_clean":"2024-07-15T07:05:05.976848+0000","last_became_active":"2024-07-15T07:04:16.929400+0000","last_became_peered":"2024-07-15T07:04:16.929400+0000","last_unstale":"2024-07-15T07:05:05.976848+0000","last_undegraded":"2024-07-15T07:05:05.976848+0000","last_fullsized":"2024-07-15T07:05:05.976848+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_clean_scrub_stamp":"2024-07-15T06:48:27.655327+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T17:33:51.920559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":52,"reported_epoch":114,"state":"active+clean","last_fresh":"2024-07-15T07:05:07.668406+0000","last_change":"2024-07-15T07:01:15.789741+0000","last_active":"2024-07-15T07:05:07.668406+0000","last_peered":"2024-07-15T07:05:07.668406+0000","last_clean":"2024-07-15T07:05:07.668406+0000","last_became_active":"2024-07-15T07:01:15.789440+0000","last_became_peered":"2024-07-15T07:01:15.789440+0000","last_unstale":"2024-07-15T07:05:07.668406+0000","last_undegraded":"2024-07-15T07:05:07.668406+0000","last_fullsized":"2024-07-15T07:05:07.668406+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T16:40:58.784616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":73,"reported_epoch":114,"state":"active+clean","last_fresh":"2024-07-15T07:05:07.670447+0000","last_change":"2024-07-15T07:03:48.041200+0000","last_active":"2024-07-15T07:05:07.670447+0000","last_peered":"2024-07-15T07:05:07.670447+0000","last_clean":"2024-07-15T07:05:07.670447+0000","last_became_active":"2024-07-15T07:03:48.038949+0000","last_became_peered":"2024-07-15T07:03:48.038949+0000","last_unstale":"2024-07-15T07:05:07.670447+0000","last_undegraded":"2024-07-15T07:05:07.670447+0000","last_fullsized":"2024-07-15T07:05:07.670447+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T17:23:57.638738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":63,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-07-15T07:05:06.450933+0000","last_change":"2024-07-15T07:04:16.906807+0000","last_active":"2024-07-15T07:05:06.450933+0000","last_peered":"2024-07-15T07:05:06.450933+0000","last_clean":"2024-07-15T07:05:06.450933+0000","last_became_active":"2024-07-15T07:04:16.905894+0000","last_became_peered":"2024-07-15T07:04:16.905894+0000","last_unstale":"2024-07-15T07:05:06.450933+0000","last_undegraded":"2024-07-15T07:05:06.450933+0000","last_fullsized":"2024-07-15T07:05:06.450933+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_clean_scrub_stamp":"2024-07-15T06:48:27.655327+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T17:27:02.666653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":69,"reported_epoch":114,"state":"active+clean","last_fresh":"2024-07-15T07:05:07.669336+0000","last_change":"2024-07-15T07:03:48.040446+0000","last_active":"2024-07-15T07:05:07.669336+0000","last_peered":"2024-07-15T07:05:07.669336+0000","last_clean":"2024-07-15T07:05:07.669336+0000","last_became_active":"2024-07-15T07:03:48.039122+0000","last_became_peered":"2024-07-15T07:03:48.039122+0000","last_unstale":"2024-07-15T07:05:07.669336+0000","last_undegraded":"2024-07-15T07:05:07.669336+0000","last_fullsized":"2024-07-15T07:05:07.669336+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T08:14:03.693610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":86,"reported_epoch":114,"state":"active+clean","last_fresh":"2024-07-15T07:05:07.668573+0000","last_change":"2024-07-15T07:04:16.914580+0000","last_active":"2024-07-15T07:05:07.668573+0000","last_peered":"2024-07-15T07:05:07.668573+0000","last_clean":"2024-07-15T07:05:07.668573+0000","last_became_active":"2024-07-15T07:04:16.913462+0000","last_became_peered":"2024-07-15T07:04:16.913462+0000","last_unstale":"2024-07-15T07:05:07.668573+0000","last_undegraded":"2024-07-15T07:05:07.668573+0000","last_fullsized":"2024-07-15T07:05:07.668573+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T15:11:13.740128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":78,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-07-15T07:05:05.966277+0000","last_change":"2024-07-15T07:03:17.336859+0000","last_active":"2024-07-15T07:05:05.966277+0000","last_peered":"2024-07-15T07:05:05.966277+0000","last_clean":"2024-07-15T07:05:05.966277+0000","last_became_active":"2024-07-15T07:03:17.336074+0000","last_became_peered":"2024-07-15T07:03:17.336074+0000","last_unstale":"2024-07-15T07:05:05.966277+0000","last_undegraded":"2024-07-15T07:05:05.966277+0000","last_fullsized":"2024-07-15T07:05:05.966277+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_clean_scrub_stamp":"2024-07-15T06:48:27.655327+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T08:46:49.778546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":25,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-07-15T07:05:05.967318+0000","last_change":"2024-07-15T07:04:16.916750+0000","last_active":"2024-07-15T07:05:05.967318+0000","last_peered":"2024-07-15T07:05:05.967318+0000","last_clean":"2024-07-15T07:05:05.967318+0000","last_became_active":"2024-07-15T07:04:16.914669+0000","last_became_peered":"2024-07-15T07:04:16.914669+0000","last_unstale":"2024-07-15T07:05:05.967318+0000","last_undegraded":"2024-07-15T07:05:05.967318+0000","last_fullsized":"2024-07-15T07:05:05.967318+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T15:24:18.547485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":83,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-07-15T07:05:05.965319+0000","last_change":"2024-07-15T07:04:16.918674+0000","last_active":"2024-07-15T07:05:05.965319+0000","last_peered":"2024-07-15T07:05:05.965319+0000","last_clean":"2024-07-15T07:05:05.965319+0000","last_became_active":"2024-07-15T07:04:16.916729+0000","last_became_peered":"2024-07-15T07:04:16.916729+0000","last_unstale":"2024-07-15T07:05:05.965319+0000","last_undegraded":"2024-07-15T07:05:05.965319+0000","last_fullsized":"2024-07-15T07:05:05.965319+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T13:00:00.925481+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"79'23","reported_seq":67,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-07-15T07:05:05.966041+0000","last_change":"2024-07-15T07:04:16.924132+0000","last_active":"2024-07-15T07:05:05.966041+0000","last_peered":"2024-07-15T07:05:05.966041+0000","last_clean":"2024-07-15T07:05:05.966041+0000","last_became_active":"2024-07-15T07:04:16.918541+0000","last_became_peered":"2024-07-15T07:04:16.918541+0000","last_unstale":"2024-07-15T07:05:05.966041+0000","last_undegraded":"2024-07-15T07:05:05.966041+0000","last_fullsized":"2024-07-15T07:05:05.966041+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T12:31:16.215154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":7,"num_read_kb":6,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":63,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-07-15T07:05:06.449618+0000","last_change":"2024-07-15T07:03:17.339339+0000","last_active":"2024-07-15T07:05:06.449618+0000","last_peered":"2024-07-15T07:05:06.449618+0000","last_clean":"2024-07-15T07:05:06.449618+0000","last_became_active":"2024-07-15T07:03:17.338443+0000","last_became_peered":"2024-07-15T07:03:17.338443+0000","last_unstale":"2024-07-15T07:05:06.449618+0000","last_undegraded":"2024-07-15T07:05:06.449618+0000","last_fullsized":"2024-07-15T07:05:06.449618+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T14:28:17.112789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":96,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-07-15T07:05:05.976608+0000","last_change":"2024-07-15T07:04:16.928817+0000","last_active":"2024-07-15T07:05:05.976608+0000","last_peered":"2024-07-15T07:05:05.976608+0000","last_clean":"2024-07-15T07:05:05.976608+0000","last_became_active":"2024-07-15T07:04:16.928091+0000","last_became_peered":"2024-07-15T07:04:16.928091+0000","last_unstale":"2024-07-15T07:05:05.976608+0000","last_undegraded":"2024-07-15T07:05:05.976608+0000","last_fullsized":"2024-07-15T07:05:05.976608+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T16:23:30.383375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":4,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":75,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-07-15T07:05:05.970989+0000","last_change":"2024-07-15T07:01:48.406717+0000","last_active":"2024-07-15T07:05:05.970989+0000","last_peered":"2024-07-15T07:05:05.970989+0000","last_clean":"2024-07-15T07:05:05.970989+0000","last_became_active":"2024-07-15T07:01:48.405962+0000","last_became_peered":"2024-07-15T07:01:48.405962+0000","last_unstale":"2024-07-15T07:05:05.970989+0000","last_undegraded":"2024-07-15T07:05:05.970989+0000","last_fullsized":"2024-07-15T07:05:05.970989+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T16:46:28.992374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":13,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-07-15T07:05:05.964830+0000","last_change":"2024-07-15T07:04:17.899894+0000","last_active":"2024-07-15T07:05:05.964830+0000","last_peered":"2024-07-15T07:05:05.964830+0000","last_clean":"2024-07-15T07:05:05.964830+0000","last_became_active":"2024-07-15T07:04:17.895749+0000","last_became_peered":"2024-07-15T07:04:17.895749+0000","last_unstale":"2024-07-15T07:05:05.964830+0000","last_undegraded":"2024-07-15T07:05:05.964830+0000","last_fullsized":"2024-07-15T07:05:05.964830+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T14:10:18.707695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":66,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-07-15T07:05:06.451092+0000","last_change":"2024-07-15T07:03:48.039874+0000","last_active":"2024-07-15T07:05:06.451092+0000","last_peered":"2024-07-15T07:05:06.451092+0000","last_clean":"2024-07-15T07:05:06.451092+0000","last_became_active":"2024-07-15T07:03:48.039007+0000","last_became_peered":"2024-07-15T07:03:48.039007+0000","last_unstale":"2024-07-15T07:05:06.451092+0000","last_undegraded":"2024-07-15T07:05:06.451092+0000","last_fullsized":"2024-07-15T07:05:06.451092+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T09:21:16.626508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":72,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-07-15T07:05:05.963396+0000","last_change":"2024-07-15T07:03:48.042847+0000","last_active":"2024-07-15T07:05:05.963396+0000","last_peered":"2024-07-15T07:05:05.963396+0000","last_clean":"2024-07-15T07:05:05.963396+0000","last_became_active":"2024-07-15T07:03:48.041903+0000","last_became_peered":"2024-07-15T07:03:48.041903+0000","last_unstale":"2024-07-15T07:05:05.963396+0000","last_undegraded":"2024-07-15T07:05:05.963396+0000","last_fullsized":"2024-07-15T07:05:05.963396+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T14:11:24.002452+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":73,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-07-15T07:05:05.966073+0000","last_change":"2024-07-15T07:01:48.425265+0000","last_active":"2024-07-15T07:05:05.966073+0000","last_peered":"2024-07-15T07:05:05.966073+0000","last_clean":"2024-07-15T07:05:05.966073+0000","last_became_active":"2024-07-15T07:01:48.424470+0000","last_became_peered":"2024-07-15T07:01:48.424470+0000","last_unstale":"2024-07-15T07:05:05.966073+0000","last_undegraded":"2024-07-15T07:05:05.966073+0000","last_fullsized":"2024-07-15T07:05:05.966073+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T07:59:37.777566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":90,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-07-15T07:05:05.963121+0000","last_change":"2024-07-15T07:03:48.039998+0000","last_active":"2024-07-15T07:05:05.963121+0000","last_peered":"2024-07-15T07:05:05.963121+0000","last_clean":"2024-07-15T07:05:05.963121+0000","last_became_active":"2024-07-15T07:03:48.037757+0000","last_became_peered":"2024-07-15T07:03:48.037757+0000","last_unstale":"2024-07-15T07:05:05.963121+0000","last_undegraded":"2024-07-15T07:05:05.963121+0000","last_fullsized":"2024-07-15T07:05:05.963121+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T12:43:48.857844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":72,"reported_epoch":114,"state":"active+clean","last_fresh":"2024-07-15T07:05:07.668772+0000","last_change":"2024-07-15T07:03:18.561774+0000","last_active":"2024-07-15T07:05:07.668772+0000","last_peered":"2024-07-15T07:05:07.668772+0000","last_clean":"2024-07-15T07:05:07.668772+0000","last_became_active":"2024-07-15T07:03:18.560819+0000","last_became_peered":"2024-07-15T07:03:18.560819+0000","last_unstale":"2024-07-15T07:05:07.668772+0000","last_undegraded":"2024-07-15T07:05:07.668772+0000","last_fullsized":"2024-07-15T07:05:07.668772+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T09:42:56.077242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":78,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-07-15T07:05:06.449606+0000","last_change":"2024-07-15T07:04:16.906773+0000","last_active":"2024-07-15T07:05:06.449606+0000","last_peered":"2024-07-15T07:05:06.449606+0000","last_clean":"2024-07-15T07:05:06.449606+0000","last_became_active":"2024-07-15T07:04:16.905874+0000","last_became_peered":"2024-07-15T07:04:16.905874+0000","last_unstale":"2024-07-15T07:05:06.449606+0000","last_undegraded":"2024-07-15T07:05:06.449606+0000","last_fullsized":"2024-07-15T07:05:06.449606+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T17:47:47.825046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":87,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-07-15T07:05:05.965433+0000","last_change":"2024-07-15T07:04:16.911610+0000","last_active":"2024-07-15T07:05:05.965433+0000","last_peered":"2024-07-15T07:05:05.965433+0000","last_clean":"2024-07-15T07:05:05.965433+0000","last_became_active":"2024-07-15T07:04:16.910695+0000","last_became_peered":"2024-07-15T07:04:16.910695+0000","last_unstale":"2024-07-15T07:05:05.965433+0000","last_undegraded":"2024-07-15T07:05:05.965433+0000","last_fullsized":"2024-07-15T07:05:05.965433+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T16:12:32.537865+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"68'2","reported_seq":93,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-07-15T07:05:05.965213+0000","last_change":"2024-07-15T07:04:16.925744+0000","last_active":"2024-07-15T07:05:05.965213+0000","last_peered":"2024-07-15T07:05:05.965213+0000","last_clean":"2024-07-15T07:05:05.965213+0000","last_became_active":"2024-07-15T07:04:16.914343+0000","last_became_peered":"2024-07-15T07:04:16.914343+0000","last_unstale":"2024-07-15T07:05:05.965213+0000","last_undegraded":"2024-07-15T07:05:05.965213+0000","last_fullsized":"2024-07-15T07:05:05.965213+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T09:36:15.360097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":68,"reported_epoch":114,"state":"active+clean","last_fresh":"2024-07-15T07:05:07.657859+0000","last_change":"2024-07-15T07:03:48.022161+0000","last_active":"2024-07-15T07:05:07.657859+0000","last_peered":"2024-07-15T07:05:07.657859+0000","last_clean":"2024-07-15T07:05:07.657859+0000","last_became_active":"2024-07-15T07:03:48.021011+0000","last_became_peered":"2024-07-15T07:03:48.021011+0000","last_unstale":"2024-07-15T07:05:07.657859+0000","last_undegraded":"2024-07-15T07:05:07.657859+0000","last_fullsized":"2024-07-15T07:05:07.657859+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T10:47:12.323014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"68'2","reported_seq":71,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-07-15T07:05:06.450845+0000","last_change":"2024-07-15T07:04:16.914630+0000","last_active":"2024-07-15T07:05:06.450845+0000","last_peered":"2024-07-15T07:05:06.450845+0000","last_clean":"2024-07-15T07:05:06.450845+0000","last_became_active":"2024-07-15T07:04:16.913785+0000","last_became_peered":"2024-07-15T07:04:16.913785+0000","last_unstale":"2024-07-15T07:05:06.450845+0000","last_undegraded":"2024-07-15T07:05:06.450845+0000","last_fullsized":"2024-07-15T07:05:06.450845+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T12:42:01.015117+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":13,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-07-15T07:05:05.964832+0000","last_change":"2024-07-15T07:04:17.899642+0000","last_active":"2024-07-15T07:05:05.964832+0000","last_peered":"2024-07-15T07:05:05.964832+0000","last_clean":"2024-07-15T07:05:05.964832+0000","last_became_active":"2024-07-15T07:04:17.898734+0000","last_became_peered":"2024-07-15T07:04:17.898734+0000","last_unstale":"2024-07-15T07:05:05.964832+0000","last_undegraded":"2024-07-15T07:05:05.964832+0000","last_fullsized":"2024-07-15T07:05:05.964832+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T13:46:46.434103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":95,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-07-15T07:05:05.971837+0000","last_change":"2024-07-15T07:04:16.911681+0000","last_active":"2024-07-15T07:05:05.971837+0000","last_peered":"2024-07-15T07:05:05.971837+0000","last_clean":"2024-07-15T07:05:05.971837+0000","last_became_active":"2024-07-15T07:04:16.910719+0000","last_became_peered":"2024-07-15T07:04:16.910719+0000","last_unstale":"2024-07-15T07:05:05.971837+0000","last_undegraded":"2024-07-15T07:05:05.971837+0000","last_fullsized":"2024-07-15T07:05:05.971837+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T11:32:38.547510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":95,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-07-15T07:05:05.964855+0000","last_change":"2024-07-15T07:04:16.908305+0000","last_active":"2024-07-15T07:05:05.964855+0000","last_peered":"2024-07-15T07:05:05.964855+0000","last_clean":"2024-07-15T07:05:05.964855+0000","last_became_active":"2024-07-15T07:04:16.905623+0000","last_became_peered":"2024-07-15T07:04:16.905623+0000","last_unstale":"2024-07-15T07:05:05.964855+0000","last_undegraded":"2024-07-15T07:05:05.964855+0000","last_fullsized":"2024-07-15T07:05:05.964855+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T08:52:02.257917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":78,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-07-15T07:05:05.972512+0000","last_change":"2024-07-15T07:04:16.911508+0000","last_active":"2024-07-15T07:05:05.972512+0000","last_peered":"2024-07-15T07:05:05.972512+0000","last_clean":"2024-07-15T07:05:05.972512+0000","last_became_active":"2024-07-15T07:04:16.910510+0000","last_became_peered":"2024-07-15T07:04:16.910510+0000","last_unstale":"2024-07-15T07:05:05.972512+0000","last_undegraded":"2024-07-15T07:05:05.972512+0000","last_fullsized":"2024-07-15T07:05:05.972512+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T13:14:14.540841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":97,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-07-15T07:05:05.963585+0000","last_change":"2024-07-15T07:03:48.040588+0000","last_active":"2024-07-15T07:05:05.963585+0000","last_peered":"2024-07-15T07:05:05.963585+0000","last_clean":"2024-07-15T07:05:05.963585+0000","last_became_active":"2024-07-15T07:03:48.037725+0000","last_became_peered":"2024-07-15T07:03:48.037725+0000","last_unstale":"2024-07-15T07:05:05.963585+0000","last_undegraded":"2024-07-15T07:05:05.963585+0000","last_fullsized":"2024-07-15T07:05:05.963585+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T16:36:18.936687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'2","reported_seq":84,"reported_epoch":114,"state":"active+clean","last_fresh":"2024-07-15T07:05:07.670426+0000","last_change":"2024-07-15T07:03:48.040605+0000","last_active":"2024-07-15T07:05:07.670426+0000","last_peered":"2024-07-15T07:05:07.670426+0000","last_clean":"2024-07-15T07:05:07.670426+0000","last_became_active":"2024-07-15T07:03:48.039707+0000","last_became_peered":"2024-07-15T07:03:48.039707+0000","last_unstale":"2024-07-15T07:05:07.670426+0000","last_undegraded":"2024-07-15T07:05:07.670426+0000","last_fullsized":"2024-07-15T07:05:07.670426+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T09:16:05.978746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":65,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-07-15T07:05:05.964651+0000","last_change":"2024-07-15T07:04:16.908639+0000","last_active":"2024-07-15T07:05:05.964651+0000","last_peered":"2024-07-15T07:05:05.964651+0000","last_clean":"2024-07-15T07:05:05.964651+0000","last_became_active":"2024-07-15T07:04:16.907763+0000","last_became_peered":"2024-07-15T07:04:16.907763+0000","last_unstale":"2024-07-15T07:05:05.964651+0000","last_undegraded":"2024-07-15T07:05:05.964651+0000","last_fullsized":"2024-07-15T07:05:05.964651+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T07:30:49.017016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"68'5","reported_seq":76,"reported_epoch":114,"state":"active+clean","last_fresh":"2024-07-15T07:05:07.668516+0000","last_change":"2024-07-15T07:03:17.333582+0000","last_active":"2024-07-15T07:05:07.668516+0000","last_peered":"2024-07-15T07:05:07.668516+0000","last_clean":"2024-07-15T07:05:07.668516+0000","last_became_active":"2024-07-15T07:03:17.332638+0000","last_became_peered":"2024-07-15T07:03:17.332638+0000","last_unstale":"2024-07-15T07:05:07.668516+0000","last_undegraded":"2024-07-15T07:05:07.668516+0000","last_fullsized":"2024-07-15T07:05:07.668516+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T18:28:19.359636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5542,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":37,"num_read_kb":39,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16626,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2387,"internal_metadata":0},"log_size":61,"ondisk_log_size":61,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234219,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":311512,"kb_used_data":1868,"kb_used_omap":1,"kb_used_meta":309630,"kb_avail":93433640,"statfs":{"total":95995035648,"available":95676047360,"internally_reserved":0,"allocated":1912832,"data_stored":1281800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317061464},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Jul 15 07:05:23 2024","interfaces":[{"interface":"back","average":{"1min":0.871,"5min":0.90000000000000002,"15min":0.92600000000000005},"min":{"1min":0.55600000000000005,"5min":0.52300000000000002,"15min":0.52300000000000002},"max":{"1min":1.2929999999999999,"5min":1.4930000000000001,"15min":1.4930000000000001},"last":1.2490000000000001},{"interface":"front","average":{"1min":0.94199999999999995,"5min":0.98099999999999998,"15min":0.98899999999999999},"min":{"1min":0.61199999999999999,"5min":0.434,"15min":0.434},"max":{"1min":1.367,"5min":2.9540000000000002,"15min":2.9540000000000002},"last":0.89200000000000002}]},{"osd":1,"last update":"Mon Jul 15 07:05:54 2024","interfaces":[{"interface":"back","average":{"1min":0.85799999999999998,"5min":0.89900000000000002,"15min":0.95199999999999996},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":1.296,"5min":1.296,"15min":1.296},"last":0.82399999999999995},{"interface":"front","average":{"1min":1.004,"5min":0.97699999999999998,"15min":1.0009999999999999},"min":{"1min":0.65300000000000002,"5min":0.55800000000000005,"15min":0.55800000000000005},"max":{"1min":1.413,"5min":1.708,"15min":1.708},"last":1.413}]},{"osd":2,"last update":"Mon Jul 15 07:05:18 2024","interfaces":[{"interface":"back","average":{"1min":0.91000000000000003,"5min":0.98899999999999999,"15min":1.002},"min":{"1min":0.67200000000000004,"5min":0.50600000000000001,"15min":0.50600000000000001},"max":{"1min":1.125,"5min":4.2450000000000001,"15min":4.2450000000000001},"last":1.1879999999999999},{"interface":"front","average":{"1min":1.018,"5min":0.92200000000000004,"15min":0.90600000000000003},"min":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"max":{"1min":2.431,"5min":2.431,"15min":2.431},"last":0.97699999999999998}]},{"osd":3,"last update":"Mon Jul 15 07:05:54 2024","interfaces":[{"interface":"back","average":{"1min":0.91100000000000003,"5min":0.94499999999999995,"15min":0.95099999999999996},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":1.2889999999999999,"5min":1.2889999999999999,"15min":1.2889999999999999},"last":1.2889999999999999},{"interface":"front","average":{"1min":1.0469999999999999,"5min":1.123,"15min":1.1359999999999999},"min":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"max":{"1min":1.8400000000000001,"5min":3.04,"15min":3.04},"last":1.1439999999999999}]},{"osd":4,"last update":"Mon Jul 15 07:05:18 2024","interfaces":[{"interface":"back","average":{"1min":0.88100000000000001,"5min":0.88100000000000001,"15min":0.88100000000000001},"min":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"max":{"1min":1.1799999999999999,"5min":1.1799999999999999,"15min":1.1799999999999999},"last":1.109},{"interface":"front","average":{"1min":1.022,"5min":1.022,"15min":1.022},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":1.2989999999999999,"5min":1.2989999999999999,"15min":1.2989999999999999},"last":1.341}]},{"osd":5,"last update":"Mon Jul 15 07:05:18 2024","interfaces":[{"interface":"back","average":{"1min":1.026,"5min":0.998,"15min":0.97699999999999998},"min":{"1min":0.47199999999999998,"5min":0.41099999999999998,"15min":0.30399999999999999},"max":{"1min":1.413,"5min":1.9019999999999999,"15min":5.141},"last":1.02},{"interface":"front","average":{"1min":0.93500000000000005,"5min":0.96799999999999997,"15min":1.022},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.39600000000000002},"max":{"1min":1.3420000000000001,"5min":1.681,"15min":4.8520000000000003},"last":1.0569999999999999}]},{"osd":6,"last update":"Mon Jul 15 07:05:18 2024","interfaces":[{"interface":"back","average":{"1min":0.98299999999999998,"5min":1.01,"15min":1.0629999999999999},"min":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.45200000000000001},"max":{"1min":1.4330000000000001,"5min":1.637,"15min":5.1870000000000003},"last":1.3759999999999999},{"interface":"front","average":{"1min":1.0309999999999999,"5min":1.0600000000000001,"15min":1.0860000000000001},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.35199999999999998},"max":{"1min":1.3380000000000001,"5min":1.542,"15min":5.0709999999999997},"last":1.224}]}]},{"osd":6,"up_from":46,"seq":197568495861,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":299220,"kb_used_data":1860,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93445932,"statfs":{"total":95995035648,"available":95688634368,"internally_reserved":0,"allocated":1904640,"data_stored":1281977,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Jul 15 07:05:23 2024","interfaces":[{"interface":"back","average":{"1min":0.89500000000000002,"5min":0.91800000000000004,"15min":0.96599999999999997},"min":{"1min":0.56499999999999995,"5min":0.54000000000000004,"15min":0.54000000000000004},"max":{"1min":1.1879999999999999,"5min":1.528,"15min":1.528},"last":1.01},{"interface":"front","average":{"1min":0.89000000000000001,"5min":0.92100000000000004,"15min":0.95599999999999996},"min":{"1min":0.67600000000000005,"5min":0.57699999999999996,"15min":0.57699999999999996},"max":{"1min":1.3520000000000001,"5min":1.3520000000000001,"15min":1.3520000000000001},"last":0.76600000000000001}]},{"osd":1,"last update":"Mon Jul 15 07:04:53 2024","interfaces":[{"interface":"back","average":{"1min":0.91600000000000004,"5min":0.91200000000000003,"15min":0.92400000000000004},"min":{"1min":0.63200000000000001,"5min":0.49299999999999999,"15min":0.49299999999999999},"max":{"1min":1.202,"5min":1.3160000000000001,"15min":1.3160000000000001},"last":0.85899999999999999},{"interface":"front","average":{"1min":0.88,"5min":1.004,"15min":1.046},"min":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"max":{"1min":1.167,"5min":1.409,"15min":1.409},"last":0.91900000000000004}]},{"osd":2,"last update":"Mon Jul 15 07:05:19 2024","interfaces":[{"interface":"back","average":{"1min":0.93999999999999995,"5min":0.92400000000000004,"15min":0.92100000000000004},"min":{"1min":0.72499999999999998,"5min":0.70599999999999996,"15min":0.70599999999999996},"max":{"1min":1.256,"5min":1.256,"15min":1.256},"last":0.80200000000000005},{"interface":"front","average":{"1min":0.96499999999999997,"5min":0.96199999999999997,"15min":0.96199999999999997},"min":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"max":{"1min":1.167,"5min":1.2450000000000001,"15min":1.2450000000000001},"last":1.101}]},{"osd":3,"last update":"Mon Jul 15 07:05:49 2024","interfaces":[{"interface":"back","average":{"1min":0.89900000000000002,"5min":0.85399999999999998,"15min":0.84599999999999997},"min":{"1min":0.68300000000000005,"5min":0.59099999999999997,"15min":0.59099999999999997},"max":{"1min":1.1200000000000001,"5min":1.129,"15min":1.129},"last":0.72699999999999998},{"interface":"front","average":{"1min":1.0680000000000001,"5min":1.02,"15min":1.012},"min":{"1min":0.70699999999999996,"5min":0.65100000000000002,"15min":0.65100000000000002},"max":{"1min":1.4179999999999999,"5min":1.4179999999999999,"15min":1.4179999999999999},"last":0.96899999999999997}]},{"osd":4,"last update":"Mon Jul 15 07:05:16 2024","interfaces":[{"interface":"back","average":{"1min":0.97099999999999997,"5min":0.97099999999999997,"15min":0.97099999999999997},"min":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"max":{"1min":1.298,"5min":1.298,"15min":1.298},"last":1.0409999999999999},{"interface":"front","average":{"1min":1.0289999999999999,"5min":1.0289999999999999,"15min":1.0289999999999999},"min":{"1min":0.499,"5min":0.499,"15min":0.499},"max":{"1min":1.3740000000000001,"5min":1.3740000000000001,"15min":1.3740000000000001},"last":1.143}]},{"osd":5,"last update":"Mon Jul 15 07:05:30 2024","interfaces":[{"interface":"back","average":{"1min":1.032,"5min":1.0089999999999999,"15min":0.98899999999999999},"min":{"1min":0.72699999999999998,"5min":0.36599999999999999,"15min":0.34399999999999997},"max":{"1min":1.448,"5min":1.673,"15min":2.6059999999999999},"last":1.2629999999999999},{"interface":"front","average":{"1min":1.036,"5min":1.0209999999999999,"15min":1.018},"min":{"1min":0.70699999999999996,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":1.405,"5min":1.7390000000000001,"15min":1.7390000000000001},"last":1.175}]},{"osd":7,"last update":"Mon Jul 15 07:05:23 2024","interfaces":[{"interface":"back","average":{"1min":1.079,"5min":1.0820000000000001,"15min":1.052},"min":{"1min":0.71499999999999997,"5min":0.42199999999999999,"15min":0.375},"max":{"1min":1.504,"5min":1.778,"15min":1.778},"last":1.2},{"interface":"front","average":{"1min":1.0429999999999999,"5min":1.0669999999999999,"15min":1.0760000000000001},"min":{"1min":0.78200000000000003,"5min":0.48199999999999998,"15min":0.48199999999999998},"max":{"1min":1.349,"5min":1.548,"15min":1.5640000000000001},"last":1.2370000000000001}]}]},{"osd":5,"up_from":38,"seq":163208757503,"num_pgs":23,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":310916,"kb_used_data":1268,"kb_used_omap":1,"kb_used_meta":309630,"kb_avail":93434236,"statfs":{"total":95995035648,"available":95676657664,"internally_reserved":0,"allocated":1298432,"data_stored":687100,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317061464},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Jul 15 07:05:24 2024","interfaces":[{"interface":"back","average":{"1min":0.80400000000000005,"5min":0.90100000000000002,"15min":0.92600000000000005},"min":{"1min":0.47699999999999998,"5min":0.47699999999999998,"15min":0.47699999999999998},"max":{"1min":1.1599999999999999,"5min":3.9929999999999999,"15min":3.9929999999999999},"last":0.99399999999999999},{"interface":"front","average":{"1min":0.92900000000000005,"5min":0.91500000000000004,"15min":0.90600000000000003},"min":{"1min":0.54800000000000004,"5min":0.503,"15min":0.503},"max":{"1min":1.379,"5min":1.379,"15min":1.379},"last":1.3}]},{"osd":1,"last update":"Mon Jul 15 07:04:56 2024","interfaces":[{"interface":"back","average":{"1min":0.86599999999999999,"5min":0.95099999999999996,"15min":0.96399999999999997},"min":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"max":{"1min":1.3029999999999999,"5min":1.3600000000000001,"15min":1.3600000000000001},"last":1.0449999999999999},{"interface":"front","average":{"1min":0.88200000000000001,"5min":0.93500000000000005,"15min":0.97299999999999998},"min":{"1min":0.58599999999999997,"5min":0.54000000000000004,"15min":0.54000000000000004},"max":{"1min":1.175,"5min":1.28,"15min":1.28},"last":1.026}]},{"osd":2,"last update":"Mon Jul 15 07:05:17 2024","interfaces":[{"interface":"back","average":{"1min":1.028,"5min":0.97399999999999998,"15min":0.96499999999999997},"min":{"1min":0.72699999999999998,"5min":0.52100000000000002,"15min":0.52100000000000002},"max":{"1min":1.347,"5min":1.5169999999999999,"15min":1.5169999999999999},"last":1.198},{"interface":"front","average":{"1min":0.94399999999999995,"5min":0.87,"15min":0.85799999999999998},"min":{"1min":0.66800000000000004,"5min":0.55400000000000005,"15min":0.55400000000000005},"max":{"1min":1.1200000000000001,"5min":1.1759999999999999,"15min":1.1759999999999999},"last":0.81899999999999995}]},{"osd":3,"last update":"Mon Jul 15 07:05:52 2024","interfaces":[{"interface":"back","average":{"1min":0.89800000000000002,"5min":1.002,"15min":1.02},"min":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"max":{"1min":1.2210000000000001,"5min":3.7890000000000001,"15min":3.7890000000000001},"last":1.0720000000000001},{"interface":"front","average":{"1min":0.91000000000000003,"5min":0.93799999999999994,"15min":0.94199999999999995},"min":{"1min":0.66300000000000003,"5min":0.64500000000000002,"15min":0.64500000000000002},"max":{"1min":1.2450000000000001,"5min":1.2450000000000001,"15min":1.2450000000000001},"last":0.92200000000000004}]},{"osd":4,"last update":"Mon Jul 15 07:05:17 2024","interfaces":[{"interface":"back","average":{"1min":1.4159999999999999,"5min":1.4159999999999999,"15min":1.4159999999999999},"min":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"max":{"1min":7.5369999999999999,"5min":7.5369999999999999,"15min":7.5369999999999999},"last":1.254},{"interface":"front","average":{"1min":1.488,"5min":1.488,"15min":1.488},"min":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"max":{"1min":7.9340000000000002,"5min":7.9340000000000002,"15min":7.9340000000000002},"last":1.109}]},{"osd":6,"last update":"Mon Jul 15 07:05:30 2024","interfaces":[{"interface":"back","average":{"1min":0.88,"5min":0.93500000000000005,"15min":0.93899999999999995},"min":{"1min":0.54600000000000004,"5min":0.39900000000000002,"15min":0.317},"max":{"1min":1.3740000000000001,"5min":2.4729999999999999,"15min":2.4729999999999999},"last":0.879},{"interface":"front","average":{"1min":0.96699999999999997,"5min":1.03,"15min":0.99299999999999999},"min":{"1min":0.65900000000000003,"5min":0.42099999999999999,"15min":0.38400000000000001},"max":{"1min":1.4179999999999999,"5min":2.5790000000000002,"15min":2.5790000000000002},"last":0.95299999999999996}]},{"osd":7,"last update":"Mon Jul 15 07:05:11 2024","interfaces":[{"interface":"back","average":{"1min":1.1799999999999999,"5min":1.1240000000000001,"15min":1.0820000000000001},"min":{"1min":0.629,"5min":0.46999999999999997,"15min":0.42099999999999999},"max":{"1min":3.1560000000000001,"5min":3.1560000000000001,"15min":3.1560000000000001},"last":1.1279999999999999},{"interface":"front","average":{"1min":1.163,"5min":1.0740000000000001,"15min":1.036},"min":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.39800000000000002},"max":{"1min":3.2629999999999999,"5min":3.2629999999999999,"15min":3.2629999999999999},"last":1.1659999999999999}]}]},{"osd":4,"up_from":110,"seq":472446402582,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28824,"kb_used_data":1288,"kb_used_omap":21,"kb_used_meta":27498,"kb_avail":93716328,"statfs":{"total":95995035648,"available":95965519872,"internally_reserved":0,"allocated":1318912,"data_stored":691685,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":21833,"internal_metadata":28158647},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Jul 15 07:05:16 2024","interfaces":[{"interface":"back","average":{"1min":0.98799999999999999,"5min":0.98799999999999999,"15min":0.98799999999999999},"min":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"max":{"1min":1.3520000000000001,"5min":1.3520000000000001,"15min":1.3520000000000001},"last":1.0009999999999999},{"interface":"front","average":{"1min":0.95699999999999996,"5min":0.95699999999999996,"15min":0.95699999999999996},"min":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"max":{"1min":1.9570000000000001,"5min":1.9570000000000001,"15min":1.9570000000000001},"last":1.03}]},{"osd":1,"last update":"Mon Jul 15 07:05:16 2024","interfaces":[{"interface":"back","average":{"1min":1.0189999999999999,"5min":1.0189999999999999,"15min":1.0189999999999999},"min":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"max":{"1min":1.2110000000000001,"5min":1.2110000000000001,"15min":1.2110000000000001},"last":1.0700000000000001},{"interface":"front","average":{"1min":1.0589999999999999,"5min":1.0589999999999999,"15min":1.0589999999999999},"min":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"max":{"1min":2.0150000000000001,"5min":2.0150000000000001,"15min":2.0150000000000001},"last":0.90400000000000003}]},{"osd":2,"last update":"Mon Jul 15 07:05:16 2024","interfaces":[{"interface":"back","average":{"1min":0.98699999999999999,"5min":0.98699999999999999,"15min":0.98699999999999999},"min":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"max":{"1min":1.367,"5min":1.367,"15min":1.367},"last":0.93899999999999995},{"interface":"front","average":{"1min":1.0880000000000001,"5min":1.0880000000000001,"15min":1.0880000000000001},"min":{"1min":0.83499999999999996,"5min":0.83499999999999996,"15min":0.83499999999999996},"max":{"1min":2.0600000000000001,"5min":2.0600000000000001,"15min":2.0600000000000001},"last":1.133}]},{"osd":3,"last update":"Mon Jul 15 07:05:16 2024","interfaces":[{"interface":"back","average":{"1min":1.077,"5min":1.077,"15min":1.077},"min":{"1min":0.67600000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"max":{"1min":1.46,"5min":1.46,"15min":1.46},"last":1.2},{"interface":"front","average":{"1min":1.1639999999999999,"5min":1.1639999999999999,"15min":1.1639999999999999},"min":{"1min":0.83999999999999997,"5min":0.83999999999999997,"15min":0.83999999999999997},"max":{"1min":1.5029999999999999,"5min":1.5029999999999999,"15min":1.5029999999999999},"last":1.179}]},{"osd":5,"last update":"Mon Jul 15 07:05:16 2024","interfaces":[{"interface":"back","average":{"1min":1.0900000000000001,"5min":1.0900000000000001,"15min":1.0900000000000001},"min":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"max":{"1min":1.603,"5min":1.603,"15min":1.603},"last":0.80600000000000005},{"interface":"front","average":{"1min":1.143,"5min":1.143,"15min":1.143},"min":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"max":{"1min":1.5609999999999999,"5min":1.5609999999999999,"15min":1.5609999999999999},"last":1.099}]},{"osd":6,"last update":"Mon Jul 15 07:05:16 2024","interfaces":[{"interface":"back","average":{"1min":0.97999999999999998,"5min":0.97999999999999998,"15min":0.97999999999999998},"min":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"max":{"1min":1.3640000000000001,"5min":1.3640000000000001,"15min":1.3640000000000001},"last":1.155},{"interface":"front","average":{"1min":1.038,"5min":1.038,"15min":1.038},"min":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"max":{"1min":1.2,"5min":1.2,"15min":1.2},"last":0.86299999999999999}]},{"osd":7,"last update":"Mon Jul 15 07:05:16 2024","interfaces":[{"interface":"back","average":{"1min":1.038,"5min":1.038,"15min":1.038},"min":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"max":{"1min":1.8839999999999999,"5min":1.8839999999999999,"15min":1.8839999999999999},"last":0.76700000000000002},{"interface":"front","average":{"1min":1.095,"5min":1.095,"15min":1.095},"min":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"max":{"1min":1.4410000000000001,"5min":1.4410000000000001,"15min":1.4410000000000001},"last":0.96899999999999997}]}]},{"osd":3,"up_from":102,"seq":438086664220,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32896,"kb_used_data":1264,"kb_used_omap":7,"kb_used_meta":31608,"kb_avail":93712256,"statfs":{"total":95995035648,"available":95961350144,"internally_reserved":0,"allocated":1294336,"data_stored":685713,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7694,"internal_metadata":32367090},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Jul 15 07:05:50 2024","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":0.67000000000000004,"15min":0.67600000000000005},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":1.1399999999999999,"5min":1.1399999999999999,"15min":1.1399999999999999},"last":0.45600000000000002},{"interface":"front","average":{"1min":0.64500000000000002,"5min":0.69299999999999995,"15min":0.70099999999999996},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.98099999999999998,"5min":0.98699999999999999,"15min":0.98699999999999999},"last":0.59999999999999998}]},{"osd":1,"last update":"Mon Jul 15 07:05:50 2024","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":0.70899999999999996,"15min":0.72399999999999998},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":1.026,"5min":1.026,"15min":1.026},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.71599999999999997,"5min":0.77200000000000002,"15min":0.78200000000000003},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.99399999999999999,"5min":1.0009999999999999,"15min":1.0009999999999999},"last":0.62}]},{"osd":2,"last update":"Mon Jul 15 07:05:50 2024","interfaces":[{"interface":"back","average":{"1min":0.71899999999999997,"5min":0.78900000000000003,"15min":0.80100000000000005},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":1.3320000000000001,"5min":1.3320000000000001,"15min":1.3320000000000001},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.72799999999999998,"5min":0.83599999999999997,"15min":0.85399999999999998},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":1.2090000000000001,"5min":1.2090000000000001,"15min":1.2090000000000001},"last":0.55400000000000005}]},{"osd":4,"last update":"Mon Jul 15 07:05:16 2024","interfaces":[{"interface":"back","average":{"1min":1.004,"5min":1.004,"15min":1.004},"min":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"max":{"1min":1.234,"5min":1.234,"15min":1.234},"last":0.73999999999999999},{"interface":"front","average":{"1min":0.997,"5min":0.997,"15min":0.997},"min":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"max":{"1min":1.306,"5min":1.306,"15min":1.306},"last":0.89500000000000002}]},{"osd":5,"last update":"Mon Jul 15 07:05:50 2024","interfaces":[{"interface":"back","average":{"1min":0.93700000000000006,"5min":1.016,"15min":1.0289999999999999},"min":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"max":{"1min":1.1519999999999999,"5min":1.3740000000000001,"15min":1.3740000000000001},"last":1.083},{"interface":"front","average":{"1min":0.95299999999999996,"5min":1.0449999999999999,"15min":1.0600000000000001},"min":{"1min":0.629,"5min":0.629,"15min":0.629},"max":{"1min":1.1890000000000001,"5min":1.544,"15min":1.544},"last":1.0669999999999999}]},{"osd":6,"last update":"Mon Jul 15 07:05:50 2024","interfaces":[{"interface":"back","average":{"1min":0.93000000000000005,"5min":1.0640000000000001,"15min":1.0860000000000001},"min":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"max":{"1min":1.266,"5min":1.4279999999999999,"15min":1.4279999999999999},"last":1.016},{"interface":"front","average":{"1min":1.018,"5min":1.0980000000000001,"15min":1.111},"min":{"1min":0.67600000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"max":{"1min":1.2729999999999999,"5min":1.3420000000000001,"15min":1.3420000000000001},"last":1.1559999999999999}]},{"osd":7,"last update":"Mon Jul 15 07:05:50 2024","interfaces":[{"interface":"back","average":{"1min":0.97099999999999997,"5min":1.0900000000000001,"15min":1.1100000000000001},"min":{"1min":0.621,"5min":0.621,"15min":0.621},"max":{"1min":1.3660000000000001,"5min":1.3999999999999999,"15min":1.3999999999999999},"last":0.98799999999999999},{"interface":"front","average":{"1min":0.98099999999999998,"5min":1.131,"15min":1.1559999999999999},"min":{"1min":0.624,"5min":0.624,"15min":0.624},"max":{"1min":1.2569999999999999,"5min":1.492,"15min":1.492},"last":1.1220000000000001}]}]},{"osd":2,"up_from":96,"seq":412316860450,"num_pgs":23,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32896,"kb_used_data":1268,"kb_used_omap":47,"kb_used_meta":31568,"kb_avail":93712256,"statfs":{"total":95995035648,"available":95961350144,"internally_reserved":0,"allocated":1298432,"data_stored":686227,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":49084,"internal_metadata":32325700},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Jul 15 07:05:19 2024","interfaces":[{"interface":"back","average":{"1min":0.65400000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"min":{"1min":0.39800000000000002,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":1.218,"5min":1.218,"15min":1.218},"last":0.82699999999999996},{"interface":"front","average":{"1min":0.76200000000000001,"5min":0.69199999999999995,"15min":0.68000000000000005},"min":{"1min":0.48699999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":1.1020000000000001,"5min":1.1020000000000001,"15min":1.1020000000000001},"last":0.53600000000000003}]},{"osd":1,"last update":"Mon Jul 15 07:05:19 2024","interfaces":[{"interface":"back","average":{"1min":0.78300000000000003,"5min":0.72699999999999998,"15min":0.71799999999999997},"min":{"1min":0.44900000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":1.722,"5min":1.722,"15min":1.722},"last":0.752},{"interface":"front","average":{"1min":0.77700000000000002,"5min":0.74299999999999999,"15min":0.73699999999999999},"min":{"1min":0.443,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":1.0740000000000001,"5min":1.0740000000000001,"15min":1.0740000000000001},"last":0.58199999999999996}]},{"osd":3,"last update":"Mon Jul 15 07:05:51 2024","interfaces":[{"interface":"back","average":{"1min":0.74299999999999999,"5min":0.78200000000000003,"15min":0.78800000000000003},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":1.0489999999999999,"5min":1.133,"15min":1.133},"last":0.66600000000000004},{"interface":"front","average":{"1min":0.78700000000000003,"5min":0.88500000000000001,"15min":0.90100000000000002},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":1.22,"5min":1.6539999999999999,"15min":1.6539999999999999},"last":0.65200000000000002}]},{"osd":4,"last update":"Mon Jul 15 07:05:19 2024","interfaces":[{"interface":"back","average":{"1min":1.0169999999999999,"5min":1.0169999999999999,"15min":1.0169999999999999},"min":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"max":{"1min":1.6819999999999999,"5min":1.6819999999999999,"15min":1.6819999999999999},"last":0.86399999999999999},{"interface":"front","average":{"1min":1.073,"5min":1.073,"15min":1.073},"min":{"1min":0.83499999999999996,"5min":0.83499999999999996,"15min":0.83499999999999996},"max":{"1min":1.4019999999999999,"5min":1.4019999999999999,"15min":1.4019999999999999},"last":0.88400000000000001}]},{"osd":5,"last update":"Mon Jul 15 07:05:19 2024","interfaces":[{"interface":"back","average":{"1min":1.0369999999999999,"5min":0.97699999999999998,"15min":0.96699999999999997},"min":{"1min":0.78600000000000003,"5min":0.68300000000000005,"15min":0.68300000000000005},"max":{"1min":1.3340000000000001,"5min":1.3340000000000001,"15min":1.3340000000000001},"last":0.78200000000000003},{"interface":"front","average":{"1min":1.0129999999999999,"5min":1,"15min":0.998},"min":{"1min":0.68500000000000005,"5min":0.61799999999999999,"15min":0.61799999999999999},"max":{"1min":1.534,"5min":1.534,"15min":1.534},"last":1.0529999999999999}]},{"osd":6,"last update":"Mon Jul 15 07:05:19 2024","interfaces":[{"interface":"back","average":{"1min":1.0289999999999999,"5min":0.995,"15min":0.98899999999999999},"min":{"1min":0.76400000000000001,"5min":0.65200000000000002,"15min":0.65200000000000002},"max":{"1min":1.6060000000000001,"5min":1.6060000000000001,"15min":1.6060000000000001},"last":0.98399999999999999},{"interface":"front","average":{"1min":1.113,"5min":1.0649999999999999,"15min":1.0569999999999999},"min":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"max":{"1min":1.4830000000000001,"5min":1.4830000000000001,"15min":1.4830000000000001},"last":0.93600000000000005}]},{"osd":7,"last update":"Mon Jul 15 07:05:19 2024","interfaces":[{"interface":"back","average":{"1min":1.155,"5min":1.1060000000000001,"15min":1.0980000000000001},"min":{"1min":0.73199999999999998,"5min":0.67000000000000004,"15min":0.67000000000000004},"max":{"1min":1.573,"5min":1.573,"15min":1.573},"last":0.96699999999999997},{"interface":"front","average":{"1min":1.081,"5min":1.0620000000000001,"15min":1.0589999999999999},"min":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"max":{"1min":1.4099999999999999,"5min":1.4690000000000001,"15min":1.4690000000000001},"last":0.90600000000000003}]}]},{"osd":1,"up_from":88,"seq":377957122100,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28792,"kb_used_data":1260,"kb_used_omap":8,"kb_used_meta":27511,"kb_avail":93716360,"statfs":{"total":95995035648,"available":95965552640,"internally_reserved":0,"allocated":1290240,"data_stored":685623,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8660,"internal_metadata":28171820},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Jul 15 07:05:54 2024","interfaces":[{"interface":"back","average":{"1min":0.64300000000000002,"5min":0.64000000000000001,"15min":0.66700000000000004},"min":{"1min":0.26000000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":1.0309999999999999,"5min":1.131,"15min":1.131},"last":0.871},{"interface":"front","average":{"1min":0.60499999999999998,"5min":0.70799999999999996,"15min":0.78200000000000003},"min":{"1min":0.32700000000000001,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":1.0289999999999999,"5min":1.131,"15min":1.131},"last":0.76900000000000002}]},{"osd":2,"last update":"Mon Jul 15 07:05:22 2024","interfaces":[{"interface":"back","average":{"1min":0.63700000000000001,"5min":0.68500000000000005,"15min":0.69299999999999995},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":1.0589999999999999,"5min":1.0589999999999999,"15min":1.0589999999999999},"last":0.54100000000000004},{"interface":"front","average":{"1min":0.73299999999999998,"5min":0.72499999999999998,"15min":0.72299999999999998},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":1.1679999999999999,"5min":1.1679999999999999,"15min":1.1679999999999999},"last":0.66400000000000003}]},{"osd":3,"last update":"Mon Jul 15 07:05:48 2024","interfaces":[{"interface":"back","average":{"1min":0.77600000000000002,"5min":0.88700000000000001,"15min":0.90500000000000003},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":1.2330000000000001,"5min":1.2330000000000001,"15min":1.2330000000000001},"last":1.0089999999999999},{"interface":"front","average":{"1min":0.78700000000000003,"5min":0.88300000000000001,"15min":0.89900000000000002},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":1.1759999999999999,"5min":1.2869999999999999,"15min":1.2869999999999999},"last":0.72199999999999998}]},{"osd":4,"last update":"Mon Jul 15 07:05:18 2024","interfaces":[{"interface":"back","average":{"1min":0.86699999999999999,"5min":0.86699999999999999,"15min":0.86699999999999999},"min":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"max":{"1min":1.262,"5min":1.262,"15min":1.262},"last":1.268},{"interface":"front","average":{"1min":1.026,"5min":1.026,"15min":1.026},"min":{"1min":0.81399999999999995,"5min":0.81399999999999995,"15min":0.81399999999999995},"max":{"1min":1.325,"5min":1.325,"15min":1.325},"last":1.077}]},{"osd":5,"last update":"Mon Jul 15 07:05:54 2024","interfaces":[{"interface":"back","average":{"1min":0.90000000000000002,"5min":0.94199999999999995,"15min":0.96599999999999997},"min":{"1min":0.54600000000000004,"5min":0.52800000000000002,"15min":0.52800000000000002},"max":{"1min":1.258,"5min":1.4299999999999999,"15min":1.4299999999999999},"last":0.90900000000000003},{"interface":"front","average":{"1min":0.94899999999999995,"5min":0.995,"15min":1.0529999999999999},"min":{"1min":0.68899999999999995,"5min":0.59899999999999998,"15min":0.59899999999999998},"max":{"1min":1.1850000000000001,"5min":1.403,"15min":1.403},"last":1.2070000000000001}]},{"osd":6,"last update":"Mon Jul 15 07:05:54 2024","interfaces":[{"interface":"back","average":{"1min":0.95899999999999996,"5min":0.999,"15min":1.0309999999999999},"min":{"1min":0.58399999999999996,"5min":0.55300000000000005,"15min":0.55300000000000005},"max":{"1min":1.357,"5min":1.381,"15min":1.381},"last":0.82799999999999996},{"interface":"front","average":{"1min":1.0009999999999999,"5min":1.0580000000000001,"15min":1.127},"min":{"1min":0.69499999999999995,"5min":0.50800000000000001,"15min":0.50800000000000001},"max":{"1min":1.2130000000000001,"5min":1.486,"15min":1.486},"last":1.321}]},{"osd":7,"last update":"Mon Jul 15 07:05:54 2024","interfaces":[{"interface":"back","average":{"1min":1.0669999999999999,"5min":1.119,"15min":1.1970000000000001},"min":{"1min":0.72599999999999998,"5min":0.61899999999999999,"15min":0.61899999999999999},"max":{"1min":1.331,"5min":1.702,"15min":1.702},"last":1.127},{"interface":"front","average":{"1min":1.014,"5min":1.097,"15min":1.1519999999999999},"min":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"max":{"1min":1.3109999999999999,"5min":1.4239999999999999,"15min":1.4239999999999999},"last":0.97599999999999998}]}]},{"osd":0,"up_from":83,"seq":356482285626,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":33480,"kb_used_data":1852,"kb_used_omap":18,"kb_used_meta":31597,"kb_avail":93711672,"statfs":{"total":95995035648,"available":95960752128,"internally_reserved":0,"allocated":1896448,"data_stored":1277743,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19384,"internal_metadata":32355400},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Jul 15 07:04:54 2024","interfaces":[{"interface":"back","average":{"1min":0.745,"5min":0.69499999999999995,"15min":0.69999999999999996},"min":{"1min":0.36799999999999999,"5min":0.311,"15min":0.311},"max":{"1min":1.216,"5min":1.216,"15min":1.216},"last":0.67900000000000005},{"interface":"front","average":{"1min":0.73899999999999999,"5min":0.70199999999999996,"15min":0.70699999999999996},"min":{"1min":0.41499999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":1.411,"5min":1.411,"15min":1.411},"last":0.71799999999999997}]},{"osd":2,"last update":"Mon Jul 15 07:05:21 2024","interfaces":[{"interface":"back","average":{"1min":0.73399999999999999,"5min":0.73699999999999999,"15min":0.73699999999999999},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":1.0329999999999999,"5min":1.0580000000000001,"15min":1.0580000000000001},"last":0.63800000000000001},{"interface":"front","average":{"1min":0.72699999999999998,"5min":0.76600000000000001,"15min":0.77200000000000002},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":1.333,"5min":1.333,"15min":1.333},"last":0.84099999999999997}]},{"osd":3,"last update":"Mon Jul 15 07:05:49 2024","interfaces":[{"interface":"back","average":{"1min":0.69199999999999995,"5min":0.79000000000000004,"15min":0.80600000000000005},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.92300000000000004,"5min":1.1359999999999999,"15min":1.1359999999999999},"last":0.81100000000000005},{"interface":"front","average":{"1min":0.747,"5min":0.76200000000000001,"15min":0.76400000000000001},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":1.0429999999999999,"5min":1.526,"15min":1.526},"last":1.016}]},{"osd":4,"last update":"Mon Jul 15 07:05:17 2024","interfaces":[{"interface":"back","average":{"1min":1.002,"5min":1.002,"15min":1.002},"min":{"1min":0.751,"5min":0.751,"15min":0.751},"max":{"1min":1.3080000000000001,"5min":1.3080000000000001,"15min":1.3080000000000001},"last":0.92000000000000004},{"interface":"front","average":{"1min":1.038,"5min":1.038,"15min":1.038},"min":{"1min":0.85199999999999998,"5min":0.85199999999999998,"15min":0.85199999999999998},"max":{"1min":1.4410000000000001,"5min":1.4410000000000001,"15min":1.4410000000000001},"last":1.256}]},{"osd":5,"last update":"Mon Jul 15 07:05:21 2024","interfaces":[{"interface":"back","average":{"1min":0.996,"5min":1.0069999999999999,"15min":1.0409999999999999},"min":{"1min":0.752,"5min":0.62,"15min":0.62},"max":{"1min":1.2529999999999999,"5min":1.49,"15min":1.49},"last":1.2190000000000001},{"interface":"front","average":{"1min":1.0680000000000001,"5min":1.0289999999999999,"15min":1.0469999999999999},"min":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"max":{"1min":1.4179999999999999,"5min":1.4179999999999999,"15min":1.4179999999999999},"last":1.409}]},{"osd":6,"last update":"Mon Jul 15 07:05:21 2024","interfaces":[{"interface":"back","average":{"1min":1.083,"5min":1.0760000000000001,"15min":1.0960000000000001},"min":{"1min":0.871,"5min":0.65100000000000002,"15min":0.65100000000000002},"max":{"1min":1.2809999999999999,"5min":1.5820000000000001,"15min":1.5820000000000001},"last":1.284},{"interface":"front","average":{"1min":1.0509999999999999,"5min":1.0620000000000001,"15min":1.089},"min":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"max":{"1min":1.498,"5min":1.5820000000000001,"15min":1.5820000000000001},"last":1.456}]},{"osd":7,"last update":"Mon Jul 15 07:05:21 2024","interfaces":[{"interface":"back","average":{"1min":1.1120000000000001,"5min":1.0900000000000001,"15min":1.107},"min":{"1min":0.68400000000000005,"5min":0.66200000000000003,"15min":0.66200000000000003},"max":{"1min":1.47,"5min":1.649,"15min":1.649},"last":1.347},{"interface":"front","average":{"1min":1.0409999999999999,"5min":1.077,"15min":1.113},"min":{"1min":0.60199999999999998,"5min":0.59599999999999997,"15min":0.59599999999999997},"max":{"1min":1.2450000000000001,"5min":1.605,"15min":1.605},"last":1.1020000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1051,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1336,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-07-15T07:05:57.765 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph pg dump --format=json 2024-07-15T07:05:58.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:05:58 smithi089 bash[79810]: cluster 2024-07-15T07:05:56.913250+0000 mgr.y (mgr.25244) 357 : cluster [DBG] pgmap v304: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:05:58.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:05:58 smithi089 bash[79810]: audit 2024-07-15T07:05:57.037410+0000 mgr.y (mgr.25244) 358 : audit [DBG] from='client.44426 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-07-15T07:05:58.873 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:05:58 smithi089 bash[82202]: cluster 2024-07-15T07:05:56.913250+0000 mgr.y (mgr.25244) 357 : cluster [DBG] pgmap v304: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:05:58.873 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:05:58 smithi089 bash[82202]: audit 2024-07-15T07:05:57.037410+0000 mgr.y (mgr.25244) 358 : audit [DBG] from='client.44426 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-07-15T07:05:59.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:05:58 smithi134 bash[67624]: cluster 2024-07-15T07:05:56.913250+0000 mgr.y (mgr.25244) 357 : cluster [DBG] pgmap v304: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:05:59.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:05:58 smithi134 bash[67624]: audit 2024-07-15T07:05:57.037410+0000 mgr.y (mgr.25244) 358 : audit [DBG] from='client.44426 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-07-15T07:06:00.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:06:00 smithi089 bash[79810]: cluster 2024-07-15T07:05:58.913941+0000 mgr.y (mgr.25244) 359 : cluster [DBG] pgmap v305: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:06:00.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:06:00 smithi089 bash[82202]: cluster 2024-07-15T07:05:58.913941+0000 mgr.y (mgr.25244) 359 : cluster [DBG] pgmap v305: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:06:01.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:06:00 smithi134 bash[67624]: cluster 2024-07-15T07:05:58.913941+0000 mgr.y (mgr.25244) 359 : cluster [DBG] pgmap v305: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:06:02.549 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T07:06:02.625 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:06:02 smithi134 bash[60729]: ts=2024-07-15T07:06:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T07:06:03.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:06:02 smithi134 bash[67624]: cluster 2024-07-15T07:06:00.914866+0000 mgr.y (mgr.25244) 360 : cluster [DBG] pgmap v306: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:06:03.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:06:02 smithi089 bash[79810]: cluster 2024-07-15T07:06:00.914866+0000 mgr.y (mgr.25244) 360 : cluster [DBG] pgmap v306: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:06:03.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:06:02 smithi089 bash[82202]: cluster 2024-07-15T07:06:00.914866+0000 mgr.y (mgr.25244) 360 : cluster [DBG] pgmap v306: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:06:04.622 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 07:06:04 smithi089 bash[72894]: ::ffff:172.21.15.134 - - [15/Jul/2024:07:06:04] "GET /metrics HTTP/1.1" 200 33974 "" "Prometheus/2.43.0" 2024-07-15T07:06:05.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:06:04 smithi134 bash[67624]: cluster 2024-07-15T07:06:02.915598+0000 mgr.y (mgr.25244) 361 : cluster [DBG] pgmap v307: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:06:05.125 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:06:04 smithi089 bash[79810]: cluster 2024-07-15T07:06:02.915598+0000 mgr.y (mgr.25244) 361 : cluster [DBG] pgmap v307: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:06:05.125 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:06:04 smithi089 bash[82202]: cluster 2024-07-15T07:06:02.915598+0000 mgr.y (mgr.25244) 361 : cluster [DBG] pgmap v307: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:06:05.900 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:06:05 smithi089 bash[79810]: audit 2024-07-15T07:06:04.797065+0000 mon.a (mon.0) 602 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:06:05.900 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:06:05 smithi089 bash[82202]: audit 2024-07-15T07:06:04.797065+0000 mon.a (mon.0) 602 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:06:06.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:06:05 smithi134 bash[67624]: audit 2024-07-15T07:06:04.797065+0000 mon.a (mon.0) 602 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:06:06.249 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-07-15T07:06:06.251 INFO:teuthology.orchestra.run.smithi089.stderr:dumped all 2024-07-15T07:06:07.003 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:06:06 smithi089 bash[79810]: cluster 2024-07-15T07:06:04.916350+0000 mgr.y (mgr.25244) 362 : cluster [DBG] pgmap v308: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:06:07.003 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:06:06 smithi089 bash[82202]: cluster 2024-07-15T07:06:04.916350+0000 mgr.y (mgr.25244) 362 : cluster [DBG] pgmap v308: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:06:07.004 INFO:teuthology.orchestra.run.smithi089.stdout:{"pg_ready":true,"pg_map":{"version":308,"stamp":"2024-07-15T07:06:04.915886+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595910,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":537,"num_read_kb":963,"num_write":300,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":1771244,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":222,"ondisk_log_size":222,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1078536,"kb_used_data":11928,"kb_used_omap":109,"kb_used_meta":1066386,"kb_avail":748882680,"statfs":{"total":767960285184,"available":766855864320,"internally_reserved":0,"allocated":12214272,"data_stored":7277868,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":111767,"internal_metadata":1091980137},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004773"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":47,"reported_epoch":114,"state":"active+clean","last_fresh":"2024-07-15T07:05:07.657645+0000","last_change":"2024-07-15T07:04:17.894309+0000","last_active":"2024-07-15T07:05:07.657645+0000","last_peered":"2024-07-15T07:05:07.657645+0000","last_clean":"2024-07-15T07:05:07.657645+0000","last_became_active":"2024-07-15T07:04:17.893694+0000","last_became_peered":"2024-07-15T07:04:17.893694+0000","last_unstale":"2024-07-15T07:05:07.657645+0000","last_undegraded":"2024-07-15T07:05:07.657645+0000","last_fullsized":"2024-07-15T07:05:07.657645+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T17:37:31.349700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":95,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-07-15T07:05:05.975211+0000","last_change":"2024-07-15T07:04:16.928586+0000","last_active":"2024-07-15T07:05:05.975211+0000","last_peered":"2024-07-15T07:05:05.975211+0000","last_clean":"2024-07-15T07:05:05.975211+0000","last_became_active":"2024-07-15T07:04:16.927823+0000","last_became_peered":"2024-07-15T07:04:16.927823+0000","last_unstale":"2024-07-15T07:05:05.975211+0000","last_undegraded":"2024-07-15T07:05:05.975211+0000","last_fullsized":"2024-07-15T07:05:05.975211+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T15:07:07.863429+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":7,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":74,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-07-15T07:05:05.972262+0000","last_change":"2024-07-15T07:03:48.026084+0000","last_active":"2024-07-15T07:05:05.972262+0000","last_peered":"2024-07-15T07:05:05.972262+0000","last_clean":"2024-07-15T07:05:05.972262+0000","last_became_active":"2024-07-15T07:03:48.024868+0000","last_became_peered":"2024-07-15T07:03:48.024868+0000","last_unstale":"2024-07-15T07:05:05.972262+0000","last_undegraded":"2024-07-15T07:05:05.972262+0000","last_fullsized":"2024-07-15T07:05:05.972262+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T16:20:53.777311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":67,"reported_epoch":114,"state":"active+clean","last_fresh":"2024-07-15T07:05:07.668195+0000","last_change":"2024-07-15T07:03:48.041561+0000","last_active":"2024-07-15T07:05:07.668195+0000","last_peered":"2024-07-15T07:05:07.668195+0000","last_clean":"2024-07-15T07:05:07.668195+0000","last_became_active":"2024-07-15T07:03:48.040745+0000","last_became_peered":"2024-07-15T07:03:48.040745+0000","last_unstale":"2024-07-15T07:05:07.668195+0000","last_undegraded":"2024-07-15T07:05:07.668195+0000","last_fullsized":"2024-07-15T07:05:07.668195+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T15:33:44.273459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":67,"reported_epoch":114,"state":"active+clean","last_fresh":"2024-07-15T07:05:07.668659+0000","last_change":"2024-07-15T07:03:48.048307+0000","last_active":"2024-07-15T07:05:07.668659+0000","last_peered":"2024-07-15T07:05:07.668659+0000","last_clean":"2024-07-15T07:05:07.668659+0000","last_became_active":"2024-07-15T07:03:48.047249+0000","last_became_peered":"2024-07-15T07:03:48.047249+0000","last_unstale":"2024-07-15T07:05:07.668659+0000","last_undegraded":"2024-07-15T07:05:07.668659+0000","last_fullsized":"2024-07-15T07:05:07.668659+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T08:21:37.555562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":74,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-07-15T07:05:05.974284+0000","last_change":"2024-07-15T07:04:16.931237+0000","last_active":"2024-07-15T07:05:05.974284+0000","last_peered":"2024-07-15T07:05:05.974284+0000","last_clean":"2024-07-15T07:05:05.974284+0000","last_became_active":"2024-07-15T07:04:16.930149+0000","last_became_peered":"2024-07-15T07:04:16.930149+0000","last_unstale":"2024-07-15T07:05:05.974284+0000","last_undegraded":"2024-07-15T07:05:05.974284+0000","last_fullsized":"2024-07-15T07:05:05.974284+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T13:02:49.562178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":27,"reported_epoch":114,"state":"active+clean","last_fresh":"2024-07-15T07:05:07.658042+0000","last_change":"2024-07-15T07:03:18.567697+0000","last_active":"2024-07-15T07:05:07.658042+0000","last_peered":"2024-07-15T07:05:07.658042+0000","last_clean":"2024-07-15T07:05:07.658042+0000","last_became_active":"2024-07-15T07:03:18.566294+0000","last_became_peered":"2024-07-15T07:03:18.566294+0000","last_unstale":"2024-07-15T07:05:07.658042+0000","last_undegraded":"2024-07-15T07:05:07.658042+0000","last_fullsized":"2024-07-15T07:05:07.658042+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T15:04:49.724233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":86,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-07-15T07:05:05.964210+0000","last_change":"2024-07-15T07:01:15.785260+0000","last_active":"2024-07-15T07:05:05.964210+0000","last_peered":"2024-07-15T07:05:05.964210+0000","last_clean":"2024-07-15T07:05:05.964210+0000","last_became_active":"2024-07-15T07:01:15.783889+0000","last_became_peered":"2024-07-15T07:01:15.783889+0000","last_unstale":"2024-07-15T07:05:05.964210+0000","last_undegraded":"2024-07-15T07:05:05.964210+0000","last_fullsized":"2024-07-15T07:05:05.964210+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T17:22:29.018297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":12,"num_read_kb":12,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":93,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-07-15T07:05:05.976353+0000","last_change":"2024-07-15T07:04:16.931104+0000","last_active":"2024-07-15T07:05:05.976353+0000","last_peered":"2024-07-15T07:05:05.976353+0000","last_clean":"2024-07-15T07:05:05.976353+0000","last_became_active":"2024-07-15T07:04:16.930135+0000","last_became_peered":"2024-07-15T07:04:16.930135+0000","last_unstale":"2024-07-15T07:05:05.976353+0000","last_undegraded":"2024-07-15T07:05:05.976353+0000","last_fullsized":"2024-07-15T07:05:05.976353+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T12:33:45.251039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":57,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-07-15T07:05:05.970906+0000","last_change":"2024-07-15T06:50:29.845162+0000","last_active":"2024-07-15T07:05:05.970906+0000","last_peered":"2024-07-15T07:05:05.970906+0000","last_clean":"2024-07-15T07:05:05.970906+0000","last_became_active":"2024-07-15T06:50:29.844204+0000","last_became_peered":"2024-07-15T06:50:29.844204+0000","last_unstale":"2024-07-15T07:05:05.970906+0000","last_undegraded":"2024-07-15T07:05:05.970906+0000","last_fullsized":"2024-07-15T07:05:05.970906+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T16:56:12.605992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":94,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-07-15T07:05:05.976174+0000","last_change":"2024-07-15T07:04:16.924775+0000","last_active":"2024-07-15T07:05:05.976174+0000","last_peered":"2024-07-15T07:05:05.976174+0000","last_clean":"2024-07-15T07:05:05.976174+0000","last_became_active":"2024-07-15T07:04:16.923784+0000","last_became_peered":"2024-07-15T07:04:16.923784+0000","last_unstale":"2024-07-15T07:05:05.976174+0000","last_undegraded":"2024-07-15T07:05:05.976174+0000","last_fullsized":"2024-07-15T07:05:05.976174+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T17:51:23.601587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":95,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-07-15T07:05:05.967573+0000","last_change":"2024-07-15T07:04:16.915477+0000","last_active":"2024-07-15T07:05:05.967573+0000","last_peered":"2024-07-15T07:05:05.967573+0000","last_clean":"2024-07-15T07:05:05.967573+0000","last_became_active":"2024-07-15T07:04:16.913839+0000","last_became_peered":"2024-07-15T07:04:16.913839+0000","last_unstale":"2024-07-15T07:05:05.967573+0000","last_undegraded":"2024-07-15T07:05:05.967573+0000","last_fullsized":"2024-07-15T07:05:05.967573+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T07:02:16.971182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":14,"reported_epoch":114,"state":"active+clean","last_fresh":"2024-07-15T07:05:06.988129+0000","last_change":"2024-07-15T07:05:06.988129+0000","last_active":"2024-07-15T07:05:06.988129+0000","last_peered":"2024-07-15T07:05:06.988129+0000","last_clean":"2024-07-15T07:05:06.988129+0000","last_became_active":"2024-07-15T07:05:06.982223+0000","last_became_peered":"2024-07-15T07:05:06.982223+0000","last_unstale":"2024-07-15T07:05:06.988129+0000","last_undegraded":"2024-07-15T07:05:06.988129+0000","last_fullsized":"2024-07-15T07:05:06.988129+0000","mapping_epoch":113,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T08:19:38.225384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":83,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-07-15T07:05:05.964238+0000","last_change":"2024-07-15T07:03:17.337106+0000","last_active":"2024-07-15T07:05:05.964238+0000","last_peered":"2024-07-15T07:05:05.964238+0000","last_clean":"2024-07-15T07:05:05.964238+0000","last_became_active":"2024-07-15T07:03:17.336185+0000","last_became_peered":"2024-07-15T07:03:17.336185+0000","last_unstale":"2024-07-15T07:05:05.964238+0000","last_undegraded":"2024-07-15T07:05:05.964238+0000","last_fullsized":"2024-07-15T07:05:05.964238+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T12:00:14.906125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":75,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-07-15T07:05:05.971882+0000","last_change":"2024-07-15T07:03:48.036322+0000","last_active":"2024-07-15T07:05:05.971882+0000","last_peered":"2024-07-15T07:05:05.971882+0000","last_clean":"2024-07-15T07:05:05.971882+0000","last_became_active":"2024-07-15T07:03:48.035590+0000","last_became_peered":"2024-07-15T07:03:48.035590+0000","last_unstale":"2024-07-15T07:05:05.971882+0000","last_undegraded":"2024-07-15T07:05:05.971882+0000","last_fullsized":"2024-07-15T07:05:05.971882+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T07:31:38.707447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":82,"reported_epoch":114,"state":"active+clean","last_fresh":"2024-07-15T07:05:07.668548+0000","last_change":"2024-07-15T07:04:16.914320+0000","last_active":"2024-07-15T07:05:07.668548+0000","last_peered":"2024-07-15T07:05:07.668548+0000","last_clean":"2024-07-15T07:05:07.668548+0000","last_became_active":"2024-07-15T07:04:16.913426+0000","last_became_peered":"2024-07-15T07:04:16.913426+0000","last_unstale":"2024-07-15T07:05:07.668548+0000","last_undegraded":"2024-07-15T07:05:07.668548+0000","last_fullsized":"2024-07-15T07:05:07.668548+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T10:33:29.340996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"68'3","reported_seq":89,"reported_epoch":114,"state":"active+clean","last_fresh":"2024-07-15T07:05:07.669696+0000","last_change":"2024-07-15T07:04:16.914557+0000","last_active":"2024-07-15T07:05:07.669696+0000","last_peered":"2024-07-15T07:05:07.669696+0000","last_clean":"2024-07-15T07:05:07.669696+0000","last_became_active":"2024-07-15T07:04:16.913557+0000","last_became_peered":"2024-07-15T07:04:16.913557+0000","last_unstale":"2024-07-15T07:05:07.669696+0000","last_undegraded":"2024-07-15T07:05:07.669696+0000","last_fullsized":"2024-07-15T07:05:07.669696+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T15:01:08.453349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":13,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-07-15T07:05:05.965499+0000","last_change":"2024-07-15T07:04:17.897049+0000","last_active":"2024-07-15T07:05:05.965499+0000","last_peered":"2024-07-15T07:05:05.965499+0000","last_clean":"2024-07-15T07:05:05.965499+0000","last_became_active":"2024-07-15T07:04:17.895834+0000","last_became_peered":"2024-07-15T07:04:17.895834+0000","last_unstale":"2024-07-15T07:05:05.965499+0000","last_undegraded":"2024-07-15T07:05:05.965499+0000","last_fullsized":"2024-07-15T07:05:05.965499+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T15:15:26.611263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":91,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-07-15T07:05:05.964527+0000","last_change":"2024-07-15T07:04:16.916338+0000","last_active":"2024-07-15T07:05:05.964527+0000","last_peered":"2024-07-15T07:05:05.964527+0000","last_clean":"2024-07-15T07:05:05.964527+0000","last_became_active":"2024-07-15T07:04:16.915416+0000","last_became_peered":"2024-07-15T07:04:16.915416+0000","last_unstale":"2024-07-15T07:05:05.964527+0000","last_undegraded":"2024-07-15T07:05:05.964527+0000","last_fullsized":"2024-07-15T07:05:05.964527+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T07:03:04.534165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":90,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-07-15T07:05:05.967781+0000","last_change":"2024-07-15T07:04:16.919574+0000","last_active":"2024-07-15T07:05:05.967781+0000","last_peered":"2024-07-15T07:05:05.967781+0000","last_clean":"2024-07-15T07:05:05.967781+0000","last_became_active":"2024-07-15T07:04:16.916288+0000","last_became_peered":"2024-07-15T07:04:16.916288+0000","last_unstale":"2024-07-15T07:05:05.967781+0000","last_undegraded":"2024-07-15T07:05:05.967781+0000","last_fullsized":"2024-07-15T07:05:05.967781+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T17:21:49.043384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":74,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-07-15T07:05:05.967299+0000","last_change":"2024-07-15T07:04:16.924979+0000","last_active":"2024-07-15T07:05:05.967299+0000","last_peered":"2024-07-15T07:05:05.967299+0000","last_clean":"2024-07-15T07:05:05.967299+0000","last_became_active":"2024-07-15T07:04:16.924217+0000","last_became_peered":"2024-07-15T07:04:16.924217+0000","last_unstale":"2024-07-15T07:05:05.967299+0000","last_undegraded":"2024-07-15T07:05:05.967299+0000","last_fullsized":"2024-07-15T07:05:05.967299+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T18:45:26.997132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":79,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-07-15T07:05:05.966025+0000","last_change":"2024-07-15T07:04:16.912026+0000","last_active":"2024-07-15T07:05:05.966025+0000","last_peered":"2024-07-15T07:05:05.966025+0000","last_clean":"2024-07-15T07:05:05.966025+0000","last_became_active":"2024-07-15T07:04:16.910983+0000","last_became_peered":"2024-07-15T07:04:16.910983+0000","last_unstale":"2024-07-15T07:05:05.966025+0000","last_undegraded":"2024-07-15T07:05:05.966025+0000","last_fullsized":"2024-07-15T07:05:05.966025+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T11:10:26.472987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":74,"reported_epoch":114,"state":"active+clean","last_fresh":"2024-07-15T07:05:07.669104+0000","last_change":"2024-07-15T07:03:48.037027+0000","last_active":"2024-07-15T07:05:07.669104+0000","last_peered":"2024-07-15T07:05:07.669104+0000","last_clean":"2024-07-15T07:05:07.669104+0000","last_became_active":"2024-07-15T07:03:48.035771+0000","last_became_peered":"2024-07-15T07:03:48.035771+0000","last_unstale":"2024-07-15T07:05:07.669104+0000","last_undegraded":"2024-07-15T07:05:07.669104+0000","last_fullsized":"2024-07-15T07:05:07.669104+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T08:50:35.241323+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":57,"reported_epoch":114,"state":"active+clean","last_fresh":"2024-07-15T07:05:07.658011+0000","last_change":"2024-07-15T07:04:16.915793+0000","last_active":"2024-07-15T07:05:07.658011+0000","last_peered":"2024-07-15T07:05:07.658011+0000","last_clean":"2024-07-15T07:05:07.658011+0000","last_became_active":"2024-07-15T07:04:16.914806+0000","last_became_peered":"2024-07-15T07:04:16.914806+0000","last_unstale":"2024-07-15T07:05:07.658011+0000","last_undegraded":"2024-07-15T07:05:07.658011+0000","last_fullsized":"2024-07-15T07:05:07.658011+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T16:42:59.639522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"79'161","reported_seq":454,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-07-15T07:05:05.973920+0000","last_change":"2024-07-15T07:01:15.788351+0000","last_active":"2024-07-15T07:05:05.973920+0000","last_peered":"2024-07-15T07:05:05.973920+0000","last_clean":"2024-07-15T07:05:05.973920+0000","last_became_active":"2024-07-15T07:01:15.787325+0000","last_became_peered":"2024-07-15T07:01:15.787325+0000","last_unstale":"2024-07-15T07:05:05.973920+0000","last_undegraded":"2024-07-15T07:05:05.973920+0000","last_fullsized":"2024-07-15T07:05:05.973920+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:42:21.582843+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:42:21.582843+0000","last_clean_scrub_stamp":"2024-07-15T06:42:21.582843+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T11:32:38.840763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":79,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-07-15T07:05:05.971764+0000","last_change":"2024-07-15T07:03:17.344493+0000","last_active":"2024-07-15T07:05:05.971764+0000","last_peered":"2024-07-15T07:05:05.971764+0000","last_clean":"2024-07-15T07:05:05.971764+0000","last_became_active":"2024-07-15T07:03:17.343699+0000","last_became_peered":"2024-07-15T07:03:17.343699+0000","last_unstale":"2024-07-15T07:05:05.971764+0000","last_undegraded":"2024-07-15T07:05:05.971764+0000","last_fullsized":"2024-07-15T07:05:05.971764+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_clean_scrub_stamp":"2024-07-15T06:48:27.655327+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T11:46:10.753577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":34,"reported_epoch":114,"state":"active+clean","last_fresh":"2024-07-15T07:05:07.669847+0000","last_change":"2024-07-15T07:03:48.042626+0000","last_active":"2024-07-15T07:05:07.669847+0000","last_peered":"2024-07-15T07:05:07.669847+0000","last_clean":"2024-07-15T07:05:07.669847+0000","last_became_active":"2024-07-15T07:03:48.041677+0000","last_became_peered":"2024-07-15T07:03:48.041677+0000","last_unstale":"2024-07-15T07:05:07.669847+0000","last_undegraded":"2024-07-15T07:05:07.669847+0000","last_fullsized":"2024-07-15T07:05:07.669847+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T14:30:01.080314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":73,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-07-15T07:05:05.965689+0000","last_change":"2024-07-15T07:04:16.918686+0000","last_active":"2024-07-15T07:05:05.965689+0000","last_peered":"2024-07-15T07:05:05.965689+0000","last_clean":"2024-07-15T07:05:05.965689+0000","last_became_active":"2024-07-15T07:04:16.917900+0000","last_became_peered":"2024-07-15T07:04:16.917900+0000","last_unstale":"2024-07-15T07:05:05.965689+0000","last_undegraded":"2024-07-15T07:05:05.965689+0000","last_fullsized":"2024-07-15T07:05:05.965689+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T09:00:26.561257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":95,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-07-15T07:05:05.973817+0000","last_change":"2024-07-15T07:01:48.416152+0000","last_active":"2024-07-15T07:05:05.973817+0000","last_peered":"2024-07-15T07:05:05.973817+0000","last_clean":"2024-07-15T07:05:05.973817+0000","last_became_active":"2024-07-15T07:01:48.415553+0000","last_became_peered":"2024-07-15T07:01:48.415553+0000","last_unstale":"2024-07-15T07:05:05.973817+0000","last_undegraded":"2024-07-15T07:05:05.973817+0000","last_fullsized":"2024-07-15T07:05:05.973817+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_clean_scrub_stamp":"2024-07-15T06:48:27.655327+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T10:50:00.600843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":82,"reported_epoch":114,"state":"active+clean","last_fresh":"2024-07-15T07:05:07.669135+0000","last_change":"2024-07-15T07:04:16.910324+0000","last_active":"2024-07-15T07:05:07.669135+0000","last_peered":"2024-07-15T07:05:07.669135+0000","last_clean":"2024-07-15T07:05:07.669135+0000","last_became_active":"2024-07-15T07:04:16.909527+0000","last_became_peered":"2024-07-15T07:04:16.909527+0000","last_unstale":"2024-07-15T07:05:07.669135+0000","last_undegraded":"2024-07-15T07:05:07.669135+0000","last_fullsized":"2024-07-15T07:05:07.669135+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T07:20:18.914775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":82,"reported_epoch":114,"state":"active+clean","last_fresh":"2024-07-15T07:05:07.668858+0000","last_change":"2024-07-15T07:03:17.331985+0000","last_active":"2024-07-15T07:05:07.668858+0000","last_peered":"2024-07-15T07:05:07.668858+0000","last_clean":"2024-07-15T07:05:07.668858+0000","last_became_active":"2024-07-15T07:03:17.331049+0000","last_became_peered":"2024-07-15T07:03:17.331049+0000","last_unstale":"2024-07-15T07:05:07.668858+0000","last_undegraded":"2024-07-15T07:05:07.668858+0000","last_fullsized":"2024-07-15T07:05:07.668858+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T07:27:20.649508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":94,"reported_epoch":114,"state":"active+clean","last_fresh":"2024-07-15T07:05:07.657352+0000","last_change":"2024-07-15T07:03:48.034592+0000","last_active":"2024-07-15T07:05:07.657352+0000","last_peered":"2024-07-15T07:05:07.657352+0000","last_clean":"2024-07-15T07:05:07.657352+0000","last_became_active":"2024-07-15T07:03:48.033864+0000","last_became_peered":"2024-07-15T07:03:48.033864+0000","last_unstale":"2024-07-15T07:05:07.657352+0000","last_undegraded":"2024-07-15T07:05:07.657352+0000","last_fullsized":"2024-07-15T07:05:07.657352+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_clean_scrub_stamp":"2024-07-15T06:48:27.655327+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T07:20:45.251256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":64,"reported_epoch":114,"state":"active+clean","last_fresh":"2024-07-15T07:05:07.671334+0000","last_change":"2024-07-15T07:03:17.330796+0000","last_active":"2024-07-15T07:05:07.671334+0000","last_peered":"2024-07-15T07:05:07.671334+0000","last_clean":"2024-07-15T07:05:07.671334+0000","last_became_active":"2024-07-15T07:03:17.329922+0000","last_became_peered":"2024-07-15T07:03:17.329922+0000","last_unstale":"2024-07-15T07:05:07.671334+0000","last_undegraded":"2024-07-15T07:05:07.671334+0000","last_fullsized":"2024-07-15T07:05:07.671334+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T08:39:59.883876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":82,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-07-15T07:05:06.450880+0000","last_change":"2024-07-15T07:01:48.417853+0000","last_active":"2024-07-15T07:05:06.450880+0000","last_peered":"2024-07-15T07:05:06.450880+0000","last_clean":"2024-07-15T07:05:06.450880+0000","last_became_active":"2024-07-15T07:01:48.413264+0000","last_became_peered":"2024-07-15T07:01:48.413264+0000","last_unstale":"2024-07-15T07:05:06.450880+0000","last_undegraded":"2024-07-15T07:05:06.450880+0000","last_fullsized":"2024-07-15T07:05:06.450880+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T07:11:58.680660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":79,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-07-15T07:05:05.971399+0000","last_change":"2024-07-15T07:01:48.412889+0000","last_active":"2024-07-15T07:05:05.971399+0000","last_peered":"2024-07-15T07:05:05.971399+0000","last_clean":"2024-07-15T07:05:05.971399+0000","last_became_active":"2024-07-15T07:01:48.411325+0000","last_became_peered":"2024-07-15T07:01:48.411325+0000","last_unstale":"2024-07-15T07:05:05.971399+0000","last_undegraded":"2024-07-15T07:05:05.971399+0000","last_fullsized":"2024-07-15T07:05:05.971399+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_clean_scrub_stamp":"2024-07-15T06:48:27.655327+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T08:05:07.677914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":61,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-07-15T07:05:05.965280+0000","last_change":"2024-07-15T07:04:16.908785+0000","last_active":"2024-07-15T07:05:05.965280+0000","last_peered":"2024-07-15T07:05:05.965280+0000","last_clean":"2024-07-15T07:05:05.965280+0000","last_became_active":"2024-07-15T07:04:16.908008+0000","last_became_peered":"2024-07-15T07:04:16.908008+0000","last_unstale":"2024-07-15T07:05:05.965280+0000","last_undegraded":"2024-07-15T07:05:05.965280+0000","last_fullsized":"2024-07-15T07:05:05.965280+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T13:32:11.951412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":62,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-07-15T07:05:05.971480+0000","last_change":"2024-07-15T07:03:48.024815+0000","last_active":"2024-07-15T07:05:05.971480+0000","last_peered":"2024-07-15T07:05:05.971480+0000","last_clean":"2024-07-15T07:05:05.971480+0000","last_became_active":"2024-07-15T07:03:48.023717+0000","last_became_peered":"2024-07-15T07:03:48.023717+0000","last_unstale":"2024-07-15T07:05:05.971480+0000","last_undegraded":"2024-07-15T07:05:05.971480+0000","last_fullsized":"2024-07-15T07:05:05.971480+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T17:02:25.635190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":126,"reported_epoch":114,"state":"active+clean","last_fresh":"2024-07-15T07:05:07.671975+0000","last_change":"2024-07-15T07:05:07.671975+0000","last_active":"2024-07-15T07:05:07.671975+0000","last_peered":"2024-07-15T07:05:07.671975+0000","last_clean":"2024-07-15T07:05:07.671975+0000","last_became_active":"2024-07-15T07:05:07.671388+0000","last_became_peered":"2024-07-15T07:05:07.671388+0000","last_unstale":"2024-07-15T07:05:07.671975+0000","last_undegraded":"2024-07-15T07:05:07.671975+0000","last_fullsized":"2024-07-15T07:05:07.671975+0000","mapping_epoch":113,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T07:23:01.512927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":88,"reported_epoch":114,"state":"active+clean","last_fresh":"2024-07-15T07:05:07.672339+0000","last_change":"2024-07-15T07:01:48.419357+0000","last_active":"2024-07-15T07:05:07.672339+0000","last_peered":"2024-07-15T07:05:07.672339+0000","last_clean":"2024-07-15T07:05:07.672339+0000","last_became_active":"2024-07-15T07:01:48.417961+0000","last_became_peered":"2024-07-15T07:01:48.417961+0000","last_unstale":"2024-07-15T07:05:07.672339+0000","last_undegraded":"2024-07-15T07:05:07.672339+0000","last_fullsized":"2024-07-15T07:05:07.672339+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_clean_scrub_stamp":"2024-07-15T06:48:27.655327+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T16:29:08.322733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":64,"reported_epoch":114,"state":"active+clean","last_fresh":"2024-07-15T07:05:07.672250+0000","last_change":"2024-07-15T07:04:16.905514+0000","last_active":"2024-07-15T07:05:07.672250+0000","last_peered":"2024-07-15T07:05:07.672250+0000","last_clean":"2024-07-15T07:05:07.672250+0000","last_became_active":"2024-07-15T07:04:16.904272+0000","last_became_peered":"2024-07-15T07:04:16.904272+0000","last_unstale":"2024-07-15T07:05:07.672250+0000","last_undegraded":"2024-07-15T07:05:07.672250+0000","last_fullsized":"2024-07-15T07:05:07.672250+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T16:34:30.504887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":64,"reported_epoch":114,"state":"active+clean","last_fresh":"2024-07-15T07:05:07.672364+0000","last_change":"2024-07-15T07:03:17.339788+0000","last_active":"2024-07-15T07:05:07.672364+0000","last_peered":"2024-07-15T07:05:07.672364+0000","last_clean":"2024-07-15T07:05:07.672364+0000","last_became_active":"2024-07-15T07:03:17.338744+0000","last_became_peered":"2024-07-15T07:03:17.338744+0000","last_unstale":"2024-07-15T07:05:07.672364+0000","last_undegraded":"2024-07-15T07:05:07.672364+0000","last_fullsized":"2024-07-15T07:05:07.672364+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T11:12:21.485090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":95,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-07-15T07:05:05.976848+0000","last_change":"2024-07-15T07:04:16.929970+0000","last_active":"2024-07-15T07:05:05.976848+0000","last_peered":"2024-07-15T07:05:05.976848+0000","last_clean":"2024-07-15T07:05:05.976848+0000","last_became_active":"2024-07-15T07:04:16.929400+0000","last_became_peered":"2024-07-15T07:04:16.929400+0000","last_unstale":"2024-07-15T07:05:05.976848+0000","last_undegraded":"2024-07-15T07:05:05.976848+0000","last_fullsized":"2024-07-15T07:05:05.976848+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_clean_scrub_stamp":"2024-07-15T06:48:27.655327+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T17:33:51.920559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":52,"reported_epoch":114,"state":"active+clean","last_fresh":"2024-07-15T07:05:07.668406+0000","last_change":"2024-07-15T07:01:15.789741+0000","last_active":"2024-07-15T07:05:07.668406+0000","last_peered":"2024-07-15T07:05:07.668406+0000","last_clean":"2024-07-15T07:05:07.668406+0000","last_became_active":"2024-07-15T07:01:15.789440+0000","last_became_peered":"2024-07-15T07:01:15.789440+0000","last_unstale":"2024-07-15T07:05:07.668406+0000","last_undegraded":"2024-07-15T07:05:07.668406+0000","last_fullsized":"2024-07-15T07:05:07.668406+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T16:40:58.784616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":73,"reported_epoch":114,"state":"active+clean","last_fresh":"2024-07-15T07:05:07.670447+0000","last_change":"2024-07-15T07:03:48.041200+0000","last_active":"2024-07-15T07:05:07.670447+0000","last_peered":"2024-07-15T07:05:07.670447+0000","last_clean":"2024-07-15T07:05:07.670447+0000","last_became_active":"2024-07-15T07:03:48.038949+0000","last_became_peered":"2024-07-15T07:03:48.038949+0000","last_unstale":"2024-07-15T07:05:07.670447+0000","last_undegraded":"2024-07-15T07:05:07.670447+0000","last_fullsized":"2024-07-15T07:05:07.670447+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T17:23:57.638738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":63,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-07-15T07:05:06.450933+0000","last_change":"2024-07-15T07:04:16.906807+0000","last_active":"2024-07-15T07:05:06.450933+0000","last_peered":"2024-07-15T07:05:06.450933+0000","last_clean":"2024-07-15T07:05:06.450933+0000","last_became_active":"2024-07-15T07:04:16.905894+0000","last_became_peered":"2024-07-15T07:04:16.905894+0000","last_unstale":"2024-07-15T07:05:06.450933+0000","last_undegraded":"2024-07-15T07:05:06.450933+0000","last_fullsized":"2024-07-15T07:05:06.450933+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_clean_scrub_stamp":"2024-07-15T06:48:27.655327+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T17:27:02.666653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":69,"reported_epoch":114,"state":"active+clean","last_fresh":"2024-07-15T07:05:07.669336+0000","last_change":"2024-07-15T07:03:48.040446+0000","last_active":"2024-07-15T07:05:07.669336+0000","last_peered":"2024-07-15T07:05:07.669336+0000","last_clean":"2024-07-15T07:05:07.669336+0000","last_became_active":"2024-07-15T07:03:48.039122+0000","last_became_peered":"2024-07-15T07:03:48.039122+0000","last_unstale":"2024-07-15T07:05:07.669336+0000","last_undegraded":"2024-07-15T07:05:07.669336+0000","last_fullsized":"2024-07-15T07:05:07.669336+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T08:14:03.693610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":86,"reported_epoch":114,"state":"active+clean","last_fresh":"2024-07-15T07:05:07.668573+0000","last_change":"2024-07-15T07:04:16.914580+0000","last_active":"2024-07-15T07:05:07.668573+0000","last_peered":"2024-07-15T07:05:07.668573+0000","last_clean":"2024-07-15T07:05:07.668573+0000","last_became_active":"2024-07-15T07:04:16.913462+0000","last_became_peered":"2024-07-15T07:04:16.913462+0000","last_unstale":"2024-07-15T07:05:07.668573+0000","last_undegraded":"2024-07-15T07:05:07.668573+0000","last_fullsized":"2024-07-15T07:05:07.668573+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T15:11:13.740128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":78,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-07-15T07:05:05.966277+0000","last_change":"2024-07-15T07:03:17.336859+0000","last_active":"2024-07-15T07:05:05.966277+0000","last_peered":"2024-07-15T07:05:05.966277+0000","last_clean":"2024-07-15T07:05:05.966277+0000","last_became_active":"2024-07-15T07:03:17.336074+0000","last_became_peered":"2024-07-15T07:03:17.336074+0000","last_unstale":"2024-07-15T07:05:05.966277+0000","last_undegraded":"2024-07-15T07:05:05.966277+0000","last_fullsized":"2024-07-15T07:05:05.966277+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:48:27.655327+0000","last_clean_scrub_stamp":"2024-07-15T06:48:27.655327+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T08:46:49.778546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":25,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-07-15T07:05:05.967318+0000","last_change":"2024-07-15T07:04:16.916750+0000","last_active":"2024-07-15T07:05:05.967318+0000","last_peered":"2024-07-15T07:05:05.967318+0000","last_clean":"2024-07-15T07:05:05.967318+0000","last_became_active":"2024-07-15T07:04:16.914669+0000","last_became_peered":"2024-07-15T07:04:16.914669+0000","last_unstale":"2024-07-15T07:05:05.967318+0000","last_undegraded":"2024-07-15T07:05:05.967318+0000","last_fullsized":"2024-07-15T07:05:05.967318+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T15:24:18.547485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":83,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-07-15T07:05:05.965319+0000","last_change":"2024-07-15T07:04:16.918674+0000","last_active":"2024-07-15T07:05:05.965319+0000","last_peered":"2024-07-15T07:05:05.965319+0000","last_clean":"2024-07-15T07:05:05.965319+0000","last_became_active":"2024-07-15T07:04:16.916729+0000","last_became_peered":"2024-07-15T07:04:16.916729+0000","last_unstale":"2024-07-15T07:05:05.965319+0000","last_undegraded":"2024-07-15T07:05:05.965319+0000","last_fullsized":"2024-07-15T07:05:05.965319+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T13:00:00.925481+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"79'23","reported_seq":67,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-07-15T07:05:05.966041+0000","last_change":"2024-07-15T07:04:16.924132+0000","last_active":"2024-07-15T07:05:05.966041+0000","last_peered":"2024-07-15T07:05:05.966041+0000","last_clean":"2024-07-15T07:05:05.966041+0000","last_became_active":"2024-07-15T07:04:16.918541+0000","last_became_peered":"2024-07-15T07:04:16.918541+0000","last_unstale":"2024-07-15T07:05:05.966041+0000","last_undegraded":"2024-07-15T07:05:05.966041+0000","last_fullsized":"2024-07-15T07:05:05.966041+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T12:31:16.215154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":7,"num_read_kb":6,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":63,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-07-15T07:05:06.449618+0000","last_change":"2024-07-15T07:03:17.339339+0000","last_active":"2024-07-15T07:05:06.449618+0000","last_peered":"2024-07-15T07:05:06.449618+0000","last_clean":"2024-07-15T07:05:06.449618+0000","last_became_active":"2024-07-15T07:03:17.338443+0000","last_became_peered":"2024-07-15T07:03:17.338443+0000","last_unstale":"2024-07-15T07:05:06.449618+0000","last_undegraded":"2024-07-15T07:05:06.449618+0000","last_fullsized":"2024-07-15T07:05:06.449618+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T14:28:17.112789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":96,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-07-15T07:05:05.976608+0000","last_change":"2024-07-15T07:04:16.928817+0000","last_active":"2024-07-15T07:05:05.976608+0000","last_peered":"2024-07-15T07:05:05.976608+0000","last_clean":"2024-07-15T07:05:05.976608+0000","last_became_active":"2024-07-15T07:04:16.928091+0000","last_became_peered":"2024-07-15T07:04:16.928091+0000","last_unstale":"2024-07-15T07:05:05.976608+0000","last_undegraded":"2024-07-15T07:05:05.976608+0000","last_fullsized":"2024-07-15T07:05:05.976608+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T16:23:30.383375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":4,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":75,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-07-15T07:05:05.970989+0000","last_change":"2024-07-15T07:01:48.406717+0000","last_active":"2024-07-15T07:05:05.970989+0000","last_peered":"2024-07-15T07:05:05.970989+0000","last_clean":"2024-07-15T07:05:05.970989+0000","last_became_active":"2024-07-15T07:01:48.405962+0000","last_became_peered":"2024-07-15T07:01:48.405962+0000","last_unstale":"2024-07-15T07:05:05.970989+0000","last_undegraded":"2024-07-15T07:05:05.970989+0000","last_fullsized":"2024-07-15T07:05:05.970989+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T16:46:28.992374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":13,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-07-15T07:05:05.964830+0000","last_change":"2024-07-15T07:04:17.899894+0000","last_active":"2024-07-15T07:05:05.964830+0000","last_peered":"2024-07-15T07:05:05.964830+0000","last_clean":"2024-07-15T07:05:05.964830+0000","last_became_active":"2024-07-15T07:04:17.895749+0000","last_became_peered":"2024-07-15T07:04:17.895749+0000","last_unstale":"2024-07-15T07:05:05.964830+0000","last_undegraded":"2024-07-15T07:05:05.964830+0000","last_fullsized":"2024-07-15T07:05:05.964830+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T14:10:18.707695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":67,"reported_epoch":114,"state":"active+clean","last_fresh":"2024-07-15T07:05:07.668748+0000","last_change":"2024-07-15T07:03:48.039874+0000","last_active":"2024-07-15T07:05:07.668748+0000","last_peered":"2024-07-15T07:05:07.668748+0000","last_clean":"2024-07-15T07:05:07.668748+0000","last_became_active":"2024-07-15T07:03:48.039007+0000","last_became_peered":"2024-07-15T07:03:48.039007+0000","last_unstale":"2024-07-15T07:05:07.668748+0000","last_undegraded":"2024-07-15T07:05:07.668748+0000","last_fullsized":"2024-07-15T07:05:07.668748+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T09:21:16.626508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":72,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-07-15T07:05:05.963396+0000","last_change":"2024-07-15T07:03:48.042847+0000","last_active":"2024-07-15T07:05:05.963396+0000","last_peered":"2024-07-15T07:05:05.963396+0000","last_clean":"2024-07-15T07:05:05.963396+0000","last_became_active":"2024-07-15T07:03:48.041903+0000","last_became_peered":"2024-07-15T07:03:48.041903+0000","last_unstale":"2024-07-15T07:05:05.963396+0000","last_undegraded":"2024-07-15T07:05:05.963396+0000","last_fullsized":"2024-07-15T07:05:05.963396+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T14:11:24.002452+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":73,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-07-15T07:05:05.966073+0000","last_change":"2024-07-15T07:01:48.425265+0000","last_active":"2024-07-15T07:05:05.966073+0000","last_peered":"2024-07-15T07:05:05.966073+0000","last_clean":"2024-07-15T07:05:05.966073+0000","last_became_active":"2024-07-15T07:01:48.424470+0000","last_became_peered":"2024-07-15T07:01:48.424470+0000","last_unstale":"2024-07-15T07:05:05.966073+0000","last_undegraded":"2024-07-15T07:05:05.966073+0000","last_fullsized":"2024-07-15T07:05:05.966073+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T07:59:37.777566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":90,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-07-15T07:05:05.963121+0000","last_change":"2024-07-15T07:03:48.039998+0000","last_active":"2024-07-15T07:05:05.963121+0000","last_peered":"2024-07-15T07:05:05.963121+0000","last_clean":"2024-07-15T07:05:05.963121+0000","last_became_active":"2024-07-15T07:03:48.037757+0000","last_became_peered":"2024-07-15T07:03:48.037757+0000","last_unstale":"2024-07-15T07:05:05.963121+0000","last_undegraded":"2024-07-15T07:05:05.963121+0000","last_fullsized":"2024-07-15T07:05:05.963121+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T12:43:48.857844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":72,"reported_epoch":114,"state":"active+clean","last_fresh":"2024-07-15T07:05:07.668772+0000","last_change":"2024-07-15T07:03:18.561774+0000","last_active":"2024-07-15T07:05:07.668772+0000","last_peered":"2024-07-15T07:05:07.668772+0000","last_clean":"2024-07-15T07:05:07.668772+0000","last_became_active":"2024-07-15T07:03:18.560819+0000","last_became_peered":"2024-07-15T07:03:18.560819+0000","last_unstale":"2024-07-15T07:05:07.668772+0000","last_undegraded":"2024-07-15T07:05:07.668772+0000","last_fullsized":"2024-07-15T07:05:07.668772+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T09:42:56.077242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":78,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-07-15T07:05:06.449606+0000","last_change":"2024-07-15T07:04:16.906773+0000","last_active":"2024-07-15T07:05:06.449606+0000","last_peered":"2024-07-15T07:05:06.449606+0000","last_clean":"2024-07-15T07:05:06.449606+0000","last_became_active":"2024-07-15T07:04:16.905874+0000","last_became_peered":"2024-07-15T07:04:16.905874+0000","last_unstale":"2024-07-15T07:05:06.449606+0000","last_undegraded":"2024-07-15T07:05:06.449606+0000","last_fullsized":"2024-07-15T07:05:06.449606+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T17:47:47.825046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":87,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-07-15T07:05:05.965433+0000","last_change":"2024-07-15T07:04:16.911610+0000","last_active":"2024-07-15T07:05:05.965433+0000","last_peered":"2024-07-15T07:05:05.965433+0000","last_clean":"2024-07-15T07:05:05.965433+0000","last_became_active":"2024-07-15T07:04:16.910695+0000","last_became_peered":"2024-07-15T07:04:16.910695+0000","last_unstale":"2024-07-15T07:05:05.965433+0000","last_undegraded":"2024-07-15T07:05:05.965433+0000","last_fullsized":"2024-07-15T07:05:05.965433+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T16:12:32.537865+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"68'2","reported_seq":93,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-07-15T07:05:05.965213+0000","last_change":"2024-07-15T07:04:16.925744+0000","last_active":"2024-07-15T07:05:05.965213+0000","last_peered":"2024-07-15T07:05:05.965213+0000","last_clean":"2024-07-15T07:05:05.965213+0000","last_became_active":"2024-07-15T07:04:16.914343+0000","last_became_peered":"2024-07-15T07:04:16.914343+0000","last_unstale":"2024-07-15T07:05:05.965213+0000","last_undegraded":"2024-07-15T07:05:05.965213+0000","last_fullsized":"2024-07-15T07:05:05.965213+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T09:36:15.360097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":68,"reported_epoch":114,"state":"active+clean","last_fresh":"2024-07-15T07:05:07.657859+0000","last_change":"2024-07-15T07:03:48.022161+0000","last_active":"2024-07-15T07:05:07.657859+0000","last_peered":"2024-07-15T07:05:07.657859+0000","last_clean":"2024-07-15T07:05:07.657859+0000","last_became_active":"2024-07-15T07:03:48.021011+0000","last_became_peered":"2024-07-15T07:03:48.021011+0000","last_unstale":"2024-07-15T07:05:07.657859+0000","last_undegraded":"2024-07-15T07:05:07.657859+0000","last_fullsized":"2024-07-15T07:05:07.657859+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T10:47:12.323014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"68'2","reported_seq":71,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-07-15T07:05:06.450845+0000","last_change":"2024-07-15T07:04:16.914630+0000","last_active":"2024-07-15T07:05:06.450845+0000","last_peered":"2024-07-15T07:05:06.450845+0000","last_clean":"2024-07-15T07:05:06.450845+0000","last_became_active":"2024-07-15T07:04:16.913785+0000","last_became_peered":"2024-07-15T07:04:16.913785+0000","last_unstale":"2024-07-15T07:05:06.450845+0000","last_undegraded":"2024-07-15T07:05:06.450845+0000","last_fullsized":"2024-07-15T07:05:06.450845+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T12:42:01.015117+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":13,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-07-15T07:05:05.964832+0000","last_change":"2024-07-15T07:04:17.899642+0000","last_active":"2024-07-15T07:05:05.964832+0000","last_peered":"2024-07-15T07:05:05.964832+0000","last_clean":"2024-07-15T07:05:05.964832+0000","last_became_active":"2024-07-15T07:04:17.898734+0000","last_became_peered":"2024-07-15T07:04:17.898734+0000","last_unstale":"2024-07-15T07:05:05.964832+0000","last_undegraded":"2024-07-15T07:05:05.964832+0000","last_fullsized":"2024-07-15T07:05:05.964832+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T13:46:46.434103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":95,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-07-15T07:05:05.971837+0000","last_change":"2024-07-15T07:04:16.911681+0000","last_active":"2024-07-15T07:05:05.971837+0000","last_peered":"2024-07-15T07:05:05.971837+0000","last_clean":"2024-07-15T07:05:05.971837+0000","last_became_active":"2024-07-15T07:04:16.910719+0000","last_became_peered":"2024-07-15T07:04:16.910719+0000","last_unstale":"2024-07-15T07:05:05.971837+0000","last_undegraded":"2024-07-15T07:05:05.971837+0000","last_fullsized":"2024-07-15T07:05:05.971837+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T11:32:38.547510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":95,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-07-15T07:05:05.964855+0000","last_change":"2024-07-15T07:04:16.908305+0000","last_active":"2024-07-15T07:05:05.964855+0000","last_peered":"2024-07-15T07:05:05.964855+0000","last_clean":"2024-07-15T07:05:05.964855+0000","last_became_active":"2024-07-15T07:04:16.905623+0000","last_became_peered":"2024-07-15T07:04:16.905623+0000","last_unstale":"2024-07-15T07:05:05.964855+0000","last_undegraded":"2024-07-15T07:05:05.964855+0000","last_fullsized":"2024-07-15T07:05:05.964855+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T08:52:02.257917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":78,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-07-15T07:05:05.972512+0000","last_change":"2024-07-15T07:04:16.911508+0000","last_active":"2024-07-15T07:05:05.972512+0000","last_peered":"2024-07-15T07:05:05.972512+0000","last_clean":"2024-07-15T07:05:05.972512+0000","last_became_active":"2024-07-15T07:04:16.910510+0000","last_became_peered":"2024-07-15T07:04:16.910510+0000","last_unstale":"2024-07-15T07:05:05.972512+0000","last_undegraded":"2024-07-15T07:05:05.972512+0000","last_fullsized":"2024-07-15T07:05:05.972512+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T13:14:14.540841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":97,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-07-15T07:05:05.963585+0000","last_change":"2024-07-15T07:03:48.040588+0000","last_active":"2024-07-15T07:05:05.963585+0000","last_peered":"2024-07-15T07:05:05.963585+0000","last_clean":"2024-07-15T07:05:05.963585+0000","last_became_active":"2024-07-15T07:03:48.037725+0000","last_became_peered":"2024-07-15T07:03:48.037725+0000","last_unstale":"2024-07-15T07:05:05.963585+0000","last_undegraded":"2024-07-15T07:05:05.963585+0000","last_fullsized":"2024-07-15T07:05:05.963585+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T16:36:18.936687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'2","reported_seq":84,"reported_epoch":114,"state":"active+clean","last_fresh":"2024-07-15T07:05:07.670426+0000","last_change":"2024-07-15T07:03:48.040605+0000","last_active":"2024-07-15T07:05:07.670426+0000","last_peered":"2024-07-15T07:05:07.670426+0000","last_clean":"2024-07-15T07:05:07.670426+0000","last_became_active":"2024-07-15T07:03:48.039707+0000","last_became_peered":"2024-07-15T07:03:48.039707+0000","last_unstale":"2024-07-15T07:05:07.670426+0000","last_undegraded":"2024-07-15T07:05:07.670426+0000","last_fullsized":"2024-07-15T07:05:07.670426+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T09:16:05.978746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":65,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-07-15T07:05:05.964651+0000","last_change":"2024-07-15T07:04:16.908639+0000","last_active":"2024-07-15T07:05:05.964651+0000","last_peered":"2024-07-15T07:05:05.964651+0000","last_clean":"2024-07-15T07:05:05.964651+0000","last_became_active":"2024-07-15T07:04:16.907763+0000","last_became_peered":"2024-07-15T07:04:16.907763+0000","last_unstale":"2024-07-15T07:05:05.964651+0000","last_undegraded":"2024-07-15T07:05:05.964651+0000","last_fullsized":"2024-07-15T07:05:05.964651+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:29.220525+0000","last_clean_scrub_stamp":"2024-07-15T06:50:29.220525+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T07:30:49.017016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"68'5","reported_seq":76,"reported_epoch":114,"state":"active+clean","last_fresh":"2024-07-15T07:05:07.668516+0000","last_change":"2024-07-15T07:03:17.333582+0000","last_active":"2024-07-15T07:05:07.668516+0000","last_peered":"2024-07-15T07:05:07.668516+0000","last_clean":"2024-07-15T07:05:07.668516+0000","last_became_active":"2024-07-15T07:03:17.332638+0000","last_became_peered":"2024-07-15T07:03:17.332638+0000","last_unstale":"2024-07-15T07:05:07.668516+0000","last_undegraded":"2024-07-15T07:05:07.668516+0000","last_fullsized":"2024-07-15T07:05:07.668516+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-15T06:50:28.211693+0000","last_clean_scrub_stamp":"2024-07-15T06:50:28.211693+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-16T18:28:19.359636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5542,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":37,"num_read_kb":39,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16626,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2387,"internal_metadata":0},"log_size":61,"ondisk_log_size":61,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234220,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":311512,"kb_used_data":1868,"kb_used_omap":1,"kb_used_meta":309630,"kb_avail":93433640,"statfs":{"total":95995035648,"available":95676047360,"internally_reserved":0,"allocated":1912832,"data_stored":1281800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317061464},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Jul 15 07:05:23 2024","interfaces":[{"interface":"back","average":{"1min":0.871,"5min":0.90000000000000002,"15min":0.92600000000000005},"min":{"1min":0.55600000000000005,"5min":0.52300000000000002,"15min":0.52300000000000002},"max":{"1min":1.2929999999999999,"5min":1.4930000000000001,"15min":1.4930000000000001},"last":0.95599999999999996},{"interface":"front","average":{"1min":0.94199999999999995,"5min":0.98099999999999998,"15min":0.98899999999999999},"min":{"1min":0.61199999999999999,"5min":0.434,"15min":0.434},"max":{"1min":1.367,"5min":2.9540000000000002,"15min":2.9540000000000002},"last":0.77300000000000002}]},{"osd":1,"last update":"Mon Jul 15 07:05:54 2024","interfaces":[{"interface":"back","average":{"1min":0.85799999999999998,"5min":0.89900000000000002,"15min":0.95199999999999996},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":1.296,"5min":1.296,"15min":1.296},"last":0.93000000000000005},{"interface":"front","average":{"1min":1.004,"5min":0.97699999999999998,"15min":1.0009999999999999},"min":{"1min":0.65300000000000002,"5min":0.55800000000000005,"15min":0.55800000000000005},"max":{"1min":1.413,"5min":1.708,"15min":1.708},"last":1.0580000000000001}]},{"osd":2,"last update":"Mon Jul 15 07:05:18 2024","interfaces":[{"interface":"back","average":{"1min":0.91000000000000003,"5min":0.98899999999999999,"15min":1.002},"min":{"1min":0.67200000000000004,"5min":0.50600000000000001,"15min":0.50600000000000001},"max":{"1min":1.125,"5min":4.2450000000000001,"15min":4.2450000000000001},"last":0.90800000000000003},{"interface":"front","average":{"1min":1.018,"5min":0.92200000000000004,"15min":0.90600000000000003},"min":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"max":{"1min":2.431,"5min":2.431,"15min":2.431},"last":1.036}]},{"osd":3,"last update":"Mon Jul 15 07:05:54 2024","interfaces":[{"interface":"back","average":{"1min":0.91100000000000003,"5min":0.94499999999999995,"15min":0.95099999999999996},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":1.2889999999999999,"5min":1.2889999999999999,"15min":1.2889999999999999},"last":0.81799999999999995},{"interface":"front","average":{"1min":1.0469999999999999,"5min":1.123,"15min":1.1359999999999999},"min":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"max":{"1min":1.8400000000000001,"5min":3.04,"15min":3.04},"last":0.872}]},{"osd":4,"last update":"Mon Jul 15 07:05:18 2024","interfaces":[{"interface":"back","average":{"1min":0.88100000000000001,"5min":0.88100000000000001,"15min":0.88100000000000001},"min":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"max":{"1min":1.1799999999999999,"5min":1.1799999999999999,"15min":1.1799999999999999},"last":0.66900000000000004},{"interface":"front","average":{"1min":1.022,"5min":1.022,"15min":1.022},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":1.2989999999999999,"5min":1.2989999999999999,"15min":1.2989999999999999},"last":1.1000000000000001}]},{"osd":5,"last update":"Mon Jul 15 07:05:18 2024","interfaces":[{"interface":"back","average":{"1min":1.026,"5min":0.998,"15min":0.97699999999999998},"min":{"1min":0.47199999999999998,"5min":0.41099999999999998,"15min":0.30399999999999999},"max":{"1min":1.413,"5min":1.9019999999999999,"15min":5.141},"last":1.0149999999999999},{"interface":"front","average":{"1min":0.93500000000000005,"5min":0.96799999999999997,"15min":1.022},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.39600000000000002},"max":{"1min":1.3420000000000001,"5min":1.681,"15min":4.8520000000000003},"last":0.72299999999999998}]},{"osd":6,"last update":"Mon Jul 15 07:05:18 2024","interfaces":[{"interface":"back","average":{"1min":0.98299999999999998,"5min":1.01,"15min":1.0629999999999999},"min":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.45200000000000001},"max":{"1min":1.4330000000000001,"5min":1.637,"15min":5.1870000000000003},"last":0.98799999999999999},{"interface":"front","average":{"1min":1.0309999999999999,"5min":1.0600000000000001,"15min":1.0860000000000001},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.35199999999999998},"max":{"1min":1.3380000000000001,"5min":1.542,"15min":5.0709999999999997},"last":0.84899999999999998}]}]},{"osd":6,"up_from":46,"seq":197568495862,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":299220,"kb_used_data":1860,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93445932,"statfs":{"total":95995035648,"available":95688634368,"internally_reserved":0,"allocated":1904640,"data_stored":1281977,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Jul 15 07:05:23 2024","interfaces":[{"interface":"back","average":{"1min":0.89500000000000002,"5min":0.91800000000000004,"15min":0.96599999999999997},"min":{"1min":0.56499999999999995,"5min":0.54000000000000004,"15min":0.54000000000000004},"max":{"1min":1.1879999999999999,"5min":1.528,"15min":1.528},"last":0.97999999999999998},{"interface":"front","average":{"1min":0.89000000000000001,"5min":0.92100000000000004,"15min":0.95599999999999996},"min":{"1min":0.67600000000000005,"5min":0.57699999999999996,"15min":0.57699999999999996},"max":{"1min":1.3520000000000001,"5min":1.3520000000000001,"15min":1.3520000000000001},"last":0.90500000000000003}]},{"osd":1,"last update":"Mon Jul 15 07:05:56 2024","interfaces":[{"interface":"back","average":{"1min":0.91900000000000004,"5min":0.91000000000000003,"15min":0.92300000000000004},"min":{"1min":0.63700000000000001,"5min":0.49299999999999999,"15min":0.49299999999999999},"max":{"1min":1.212,"5min":1.3160000000000001,"15min":1.3160000000000001},"last":1.0129999999999999},{"interface":"front","average":{"1min":0.92700000000000005,"5min":0.97499999999999998,"15min":1.0369999999999999},"min":{"1min":0.60799999999999998,"5min":0.58599999999999997,"15min":0.58599999999999997},"max":{"1min":1.3049999999999999,"5min":1.409,"15min":1.409},"last":0.85399999999999998}]},{"osd":2,"last update":"Mon Jul 15 07:05:19 2024","interfaces":[{"interface":"back","average":{"1min":0.93999999999999995,"5min":0.92400000000000004,"15min":0.92100000000000004},"min":{"1min":0.72499999999999998,"5min":0.70599999999999996,"15min":0.70599999999999996},"max":{"1min":1.256,"5min":1.256,"15min":1.256},"last":1.1240000000000001},{"interface":"front","average":{"1min":0.96499999999999997,"5min":0.96199999999999997,"15min":0.96199999999999997},"min":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"max":{"1min":1.167,"5min":1.2450000000000001,"15min":1.2450000000000001},"last":0.74399999999999999}]},{"osd":3,"last update":"Mon Jul 15 07:05:49 2024","interfaces":[{"interface":"back","average":{"1min":0.89900000000000002,"5min":0.85399999999999998,"15min":0.84599999999999997},"min":{"1min":0.68300000000000005,"5min":0.59099999999999997,"15min":0.59099999999999997},"max":{"1min":1.1200000000000001,"5min":1.129,"15min":1.129},"last":0.71899999999999997},{"interface":"front","average":{"1min":1.0680000000000001,"5min":1.02,"15min":1.012},"min":{"1min":0.70699999999999996,"5min":0.65100000000000002,"15min":0.65100000000000002},"max":{"1min":1.4179999999999999,"5min":1.4179999999999999,"15min":1.4179999999999999},"last":1.147}]},{"osd":4,"last update":"Mon Jul 15 07:05:16 2024","interfaces":[{"interface":"back","average":{"1min":0.97099999999999997,"5min":0.97099999999999997,"15min":0.97099999999999997},"min":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"max":{"1min":1.298,"5min":1.298,"15min":1.298},"last":0.93899999999999995},{"interface":"front","average":{"1min":1.0289999999999999,"5min":1.0289999999999999,"15min":1.0289999999999999},"min":{"1min":0.499,"5min":0.499,"15min":0.499},"max":{"1min":1.3740000000000001,"5min":1.3740000000000001,"15min":1.3740000000000001},"last":1.0429999999999999}]},{"osd":5,"last update":"Mon Jul 15 07:05:30 2024","interfaces":[{"interface":"back","average":{"1min":1.032,"5min":1.0089999999999999,"15min":0.98899999999999999},"min":{"1min":0.72699999999999998,"5min":0.36599999999999999,"15min":0.34399999999999997},"max":{"1min":1.448,"5min":1.673,"15min":2.6059999999999999},"last":0.63500000000000001},{"interface":"front","average":{"1min":1.036,"5min":1.0209999999999999,"15min":1.018},"min":{"1min":0.70699999999999996,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":1.405,"5min":1.7390000000000001,"15min":1.7390000000000001},"last":0.82499999999999996}]},{"osd":7,"last update":"Mon Jul 15 07:05:23 2024","interfaces":[{"interface":"back","average":{"1min":1.079,"5min":1.0820000000000001,"15min":1.052},"min":{"1min":0.71499999999999997,"5min":0.42199999999999999,"15min":0.375},"max":{"1min":1.504,"5min":1.778,"15min":1.778},"last":1.1699999999999999},{"interface":"front","average":{"1min":1.0429999999999999,"5min":1.0669999999999999,"15min":1.0760000000000001},"min":{"1min":0.78200000000000003,"5min":0.48199999999999998,"15min":0.48199999999999998},"max":{"1min":1.349,"5min":1.548,"15min":1.5640000000000001},"last":1.091}]}]},{"osd":5,"up_from":38,"seq":163208757504,"num_pgs":23,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":310916,"kb_used_data":1268,"kb_used_omap":1,"kb_used_meta":309630,"kb_avail":93434236,"statfs":{"total":95995035648,"available":95676657664,"internally_reserved":0,"allocated":1298432,"data_stored":687100,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317061464},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Jul 15 07:05:24 2024","interfaces":[{"interface":"back","average":{"1min":0.80400000000000005,"5min":0.90100000000000002,"15min":0.92600000000000005},"min":{"1min":0.47699999999999998,"5min":0.47699999999999998,"15min":0.47699999999999998},"max":{"1min":1.1599999999999999,"5min":3.9929999999999999,"15min":3.9929999999999999},"last":1},{"interface":"front","average":{"1min":0.92900000000000005,"5min":0.91500000000000004,"15min":0.90600000000000003},"min":{"1min":0.54800000000000004,"5min":0.503,"15min":0.503},"max":{"1min":1.379,"5min":1.379,"15min":1.379},"last":0.70899999999999996}]},{"osd":1,"last update":"Mon Jul 15 07:05:56 2024","interfaces":[{"interface":"back","average":{"1min":0.92200000000000004,"5min":0.94099999999999995,"15min":0.96099999999999997},"min":{"1min":0.57299999999999995,"5min":0.54500000000000004,"15min":0.54500000000000004},"max":{"1min":1.252,"5min":1.3600000000000001,"15min":1.3600000000000001},"last":0.88400000000000001},{"interface":"front","average":{"1min":0.92700000000000005,"5min":0.92200000000000004,"15min":0.96899999999999997},"min":{"1min":0.60499999999999998,"5min":0.54000000000000004,"15min":0.54000000000000004},"max":{"1min":1.1930000000000001,"5min":1.28,"15min":1.28},"last":1.0569999999999999}]},{"osd":2,"last update":"Mon Jul 15 07:05:17 2024","interfaces":[{"interface":"back","average":{"1min":1.028,"5min":0.97399999999999998,"15min":0.96499999999999997},"min":{"1min":0.72699999999999998,"5min":0.52100000000000002,"15min":0.52100000000000002},"max":{"1min":1.347,"5min":1.5169999999999999,"15min":1.5169999999999999},"last":0.96099999999999997},{"interface":"front","average":{"1min":0.94399999999999995,"5min":0.87,"15min":0.85799999999999998},"min":{"1min":0.66800000000000004,"5min":0.55400000000000005,"15min":0.55400000000000005},"max":{"1min":1.1200000000000001,"5min":1.1759999999999999,"15min":1.1759999999999999},"last":1.206}]},{"osd":3,"last update":"Mon Jul 15 07:05:52 2024","interfaces":[{"interface":"back","average":{"1min":0.89800000000000002,"5min":1.002,"15min":1.02},"min":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"max":{"1min":1.2210000000000001,"5min":3.7890000000000001,"15min":3.7890000000000001},"last":0.748},{"interface":"front","average":{"1min":0.91000000000000003,"5min":0.93799999999999994,"15min":0.94199999999999995},"min":{"1min":0.66300000000000003,"5min":0.64500000000000002,"15min":0.64500000000000002},"max":{"1min":1.2450000000000001,"5min":1.2450000000000001,"15min":1.2450000000000001},"last":1.304}]},{"osd":4,"last update":"Mon Jul 15 07:05:17 2024","interfaces":[{"interface":"back","average":{"1min":1.4159999999999999,"5min":1.4159999999999999,"15min":1.4159999999999999},"min":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"max":{"1min":7.5369999999999999,"5min":7.5369999999999999,"15min":7.5369999999999999},"last":0.80200000000000005},{"interface":"front","average":{"1min":1.488,"5min":1.488,"15min":1.488},"min":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"max":{"1min":7.9340000000000002,"5min":7.9340000000000002,"15min":7.9340000000000002},"last":1.3819999999999999}]},{"osd":6,"last update":"Mon Jul 15 07:05:30 2024","interfaces":[{"interface":"back","average":{"1min":0.88,"5min":0.93500000000000005,"15min":0.93899999999999995},"min":{"1min":0.54600000000000004,"5min":0.39900000000000002,"15min":0.317},"max":{"1min":1.3740000000000001,"5min":2.4729999999999999,"15min":2.4729999999999999},"last":1.119},{"interface":"front","average":{"1min":0.96699999999999997,"5min":1.03,"15min":0.99299999999999999},"min":{"1min":0.65900000000000003,"5min":0.42099999999999999,"15min":0.38400000000000001},"max":{"1min":1.4179999999999999,"5min":2.5790000000000002,"15min":2.5790000000000002},"last":1.145}]},{"osd":7,"last update":"Mon Jul 15 07:05:11 2024","interfaces":[{"interface":"back","average":{"1min":1.1799999999999999,"5min":1.1240000000000001,"15min":1.0820000000000001},"min":{"1min":0.629,"5min":0.46999999999999997,"15min":0.42099999999999999},"max":{"1min":3.1560000000000001,"5min":3.1560000000000001,"15min":3.1560000000000001},"last":1.038},{"interface":"front","average":{"1min":1.163,"5min":1.0740000000000001,"15min":1.036},"min":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.39800000000000002},"max":{"1min":3.2629999999999999,"5min":3.2629999999999999,"15min":3.2629999999999999},"last":1.454}]}]},{"osd":4,"up_from":110,"seq":472446402583,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28824,"kb_used_data":1288,"kb_used_omap":21,"kb_used_meta":27498,"kb_avail":93716328,"statfs":{"total":95995035648,"available":95965519872,"internally_reserved":0,"allocated":1318912,"data_stored":691685,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":21833,"internal_metadata":28158647},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Jul 15 07:05:16 2024","interfaces":[{"interface":"back","average":{"1min":0.98799999999999999,"5min":0.98799999999999999,"15min":0.98799999999999999},"min":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"max":{"1min":1.3520000000000001,"5min":1.3520000000000001,"15min":1.3520000000000001},"last":0.79300000000000004},{"interface":"front","average":{"1min":0.95699999999999996,"5min":0.95699999999999996,"15min":0.95699999999999996},"min":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"max":{"1min":1.9570000000000001,"5min":1.9570000000000001,"15min":1.9570000000000001},"last":1.1539999999999999}]},{"osd":1,"last update":"Mon Jul 15 07:05:16 2024","interfaces":[{"interface":"back","average":{"1min":1.0189999999999999,"5min":1.0189999999999999,"15min":1.0189999999999999},"min":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"max":{"1min":1.2110000000000001,"5min":1.2110000000000001,"15min":1.2110000000000001},"last":1.121},{"interface":"front","average":{"1min":1.0589999999999999,"5min":1.0589999999999999,"15min":1.0589999999999999},"min":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"max":{"1min":2.0150000000000001,"5min":2.0150000000000001,"15min":2.0150000000000001},"last":1.056}]},{"osd":2,"last update":"Mon Jul 15 07:05:16 2024","interfaces":[{"interface":"back","average":{"1min":0.98699999999999999,"5min":0.98699999999999999,"15min":0.98699999999999999},"min":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"max":{"1min":1.367,"5min":1.367,"15min":1.367},"last":0.90300000000000002},{"interface":"front","average":{"1min":1.0880000000000001,"5min":1.0880000000000001,"15min":1.0880000000000001},"min":{"1min":0.83499999999999996,"5min":0.83499999999999996,"15min":0.83499999999999996},"max":{"1min":2.0600000000000001,"5min":2.0600000000000001,"15min":2.0600000000000001},"last":0.93300000000000005}]},{"osd":3,"last update":"Mon Jul 15 07:05:16 2024","interfaces":[{"interface":"back","average":{"1min":1.077,"5min":1.077,"15min":1.077},"min":{"1min":0.67600000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"max":{"1min":1.46,"5min":1.46,"15min":1.46},"last":0.75600000000000001},{"interface":"front","average":{"1min":1.1639999999999999,"5min":1.1639999999999999,"15min":1.1639999999999999},"min":{"1min":0.83999999999999997,"5min":0.83999999999999997,"15min":0.83999999999999997},"max":{"1min":1.5029999999999999,"5min":1.5029999999999999,"15min":1.5029999999999999},"last":1.0229999999999999}]},{"osd":5,"last update":"Mon Jul 15 07:05:16 2024","interfaces":[{"interface":"back","average":{"1min":1.0900000000000001,"5min":1.0900000000000001,"15min":1.0900000000000001},"min":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"max":{"1min":1.603,"5min":1.603,"15min":1.603},"last":1.2130000000000001},{"interface":"front","average":{"1min":1.143,"5min":1.143,"15min":1.143},"min":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"max":{"1min":1.5609999999999999,"5min":1.5609999999999999,"15min":1.5609999999999999},"last":1.1859999999999999}]},{"osd":6,"last update":"Mon Jul 15 07:05:16 2024","interfaces":[{"interface":"back","average":{"1min":0.97999999999999998,"5min":0.97999999999999998,"15min":0.97999999999999998},"min":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"max":{"1min":1.3640000000000001,"5min":1.3640000000000001,"15min":1.3640000000000001},"last":0.87},{"interface":"front","average":{"1min":1.038,"5min":1.038,"15min":1.038},"min":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"max":{"1min":1.2,"5min":1.2,"15min":1.2},"last":1.0880000000000001}]},{"osd":7,"last update":"Mon Jul 15 07:05:16 2024","interfaces":[{"interface":"back","average":{"1min":1.038,"5min":1.038,"15min":1.038},"min":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"max":{"1min":1.8839999999999999,"5min":1.8839999999999999,"15min":1.8839999999999999},"last":0.82999999999999996},{"interface":"front","average":{"1min":1.095,"5min":1.095,"15min":1.095},"min":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"max":{"1min":1.4410000000000001,"5min":1.4410000000000001,"15min":1.4410000000000001},"last":0.98499999999999999}]}]},{"osd":3,"up_from":102,"seq":438086664221,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32896,"kb_used_data":1264,"kb_used_omap":7,"kb_used_meta":31608,"kb_avail":93712256,"statfs":{"total":95995035648,"available":95961350144,"internally_reserved":0,"allocated":1294336,"data_stored":685713,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7694,"internal_metadata":32367090},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Jul 15 07:05:50 2024","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":0.67000000000000004,"15min":0.67600000000000005},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":1.1399999999999999,"5min":1.1399999999999999,"15min":1.1399999999999999},"last":0.56599999999999995},{"interface":"front","average":{"1min":0.64500000000000002,"5min":0.69299999999999995,"15min":0.70099999999999996},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.98099999999999998,"5min":0.98699999999999999,"15min":0.98699999999999999},"last":1.1719999999999999}]},{"osd":1,"last update":"Mon Jul 15 07:05:50 2024","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":0.70899999999999996,"15min":0.72399999999999998},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":1.026,"5min":1.026,"15min":1.026},"last":0.60999999999999999},{"interface":"front","average":{"1min":0.71599999999999997,"5min":0.77200000000000002,"15min":0.78200000000000003},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.99399999999999999,"5min":1.0009999999999999,"15min":1.0009999999999999},"last":0.67000000000000004}]},{"osd":2,"last update":"Mon Jul 15 07:05:50 2024","interfaces":[{"interface":"back","average":{"1min":0.71899999999999997,"5min":0.78900000000000003,"15min":0.80100000000000005},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":1.3320000000000001,"5min":1.3320000000000001,"15min":1.3320000000000001},"last":0.90000000000000002},{"interface":"front","average":{"1min":0.72799999999999998,"5min":0.83599999999999997,"15min":0.85399999999999998},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":1.2090000000000001,"5min":1.2090000000000001,"15min":1.2090000000000001},"last":0.81599999999999995}]},{"osd":4,"last update":"Mon Jul 15 07:05:16 2024","interfaces":[{"interface":"back","average":{"1min":1.004,"5min":1.004,"15min":1.004},"min":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"max":{"1min":1.234,"5min":1.234,"15min":1.234},"last":0.996},{"interface":"front","average":{"1min":0.997,"5min":0.997,"15min":0.997},"min":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"max":{"1min":1.306,"5min":1.306,"15min":1.306},"last":0.77100000000000002}]},{"osd":5,"last update":"Mon Jul 15 07:05:50 2024","interfaces":[{"interface":"back","average":{"1min":0.93700000000000006,"5min":1.016,"15min":1.0289999999999999},"min":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"max":{"1min":1.1519999999999999,"5min":1.3740000000000001,"15min":1.3740000000000001},"last":1.0780000000000001},{"interface":"front","average":{"1min":0.95299999999999996,"5min":1.0449999999999999,"15min":1.0600000000000001},"min":{"1min":0.629,"5min":0.629,"15min":0.629},"max":{"1min":1.1890000000000001,"5min":1.544,"15min":1.544},"last":1.0289999999999999}]},{"osd":6,"last update":"Mon Jul 15 07:05:50 2024","interfaces":[{"interface":"back","average":{"1min":0.93000000000000005,"5min":1.0640000000000001,"15min":1.0860000000000001},"min":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"max":{"1min":1.266,"5min":1.4279999999999999,"15min":1.4279999999999999},"last":0.93799999999999994},{"interface":"front","average":{"1min":1.018,"5min":1.0980000000000001,"15min":1.111},"min":{"1min":0.67600000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"max":{"1min":1.2729999999999999,"5min":1.3420000000000001,"15min":1.3420000000000001},"last":0.96599999999999997}]},{"osd":7,"last update":"Mon Jul 15 07:05:50 2024","interfaces":[{"interface":"back","average":{"1min":0.97099999999999997,"5min":1.0900000000000001,"15min":1.1100000000000001},"min":{"1min":0.621,"5min":0.621,"15min":0.621},"max":{"1min":1.3660000000000001,"5min":1.3999999999999999,"15min":1.3999999999999999},"last":1.111},{"interface":"front","average":{"1min":0.98099999999999998,"5min":1.131,"15min":1.1559999999999999},"min":{"1min":0.624,"5min":0.624,"15min":0.624},"max":{"1min":1.2569999999999999,"5min":1.492,"15min":1.492},"last":1.056}]}]},{"osd":2,"up_from":96,"seq":412316860451,"num_pgs":23,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32896,"kb_used_data":1268,"kb_used_omap":47,"kb_used_meta":31568,"kb_avail":93712256,"statfs":{"total":95995035648,"available":95961350144,"internally_reserved":0,"allocated":1298432,"data_stored":686227,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":49084,"internal_metadata":32325700},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Jul 15 07:05:19 2024","interfaces":[{"interface":"back","average":{"1min":0.65400000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"min":{"1min":0.39800000000000002,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":1.218,"5min":1.218,"15min":1.218},"last":0.82699999999999996},{"interface":"front","average":{"1min":0.76200000000000001,"5min":0.69199999999999995,"15min":0.68000000000000005},"min":{"1min":0.48699999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":1.1020000000000001,"5min":1.1020000000000001,"15min":1.1020000000000001},"last":0.53600000000000003}]},{"osd":1,"last update":"Mon Jul 15 07:05:19 2024","interfaces":[{"interface":"back","average":{"1min":0.78300000000000003,"5min":0.72699999999999998,"15min":0.71799999999999997},"min":{"1min":0.44900000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":1.722,"5min":1.722,"15min":1.722},"last":0.752},{"interface":"front","average":{"1min":0.77700000000000002,"5min":0.74299999999999999,"15min":0.73699999999999999},"min":{"1min":0.443,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":1.0740000000000001,"5min":1.0740000000000001,"15min":1.0740000000000001},"last":0.58199999999999996}]},{"osd":3,"last update":"Mon Jul 15 07:05:51 2024","interfaces":[{"interface":"back","average":{"1min":0.74299999999999999,"5min":0.78200000000000003,"15min":0.78800000000000003},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":1.0489999999999999,"5min":1.133,"15min":1.133},"last":0.66600000000000004},{"interface":"front","average":{"1min":0.78700000000000003,"5min":0.88500000000000001,"15min":0.90100000000000002},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":1.22,"5min":1.6539999999999999,"15min":1.6539999999999999},"last":0.65200000000000002}]},{"osd":4,"last update":"Mon Jul 15 07:05:19 2024","interfaces":[{"interface":"back","average":{"1min":1.0169999999999999,"5min":1.0169999999999999,"15min":1.0169999999999999},"min":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"max":{"1min":1.6819999999999999,"5min":1.6819999999999999,"15min":1.6819999999999999},"last":0.86399999999999999},{"interface":"front","average":{"1min":1.073,"5min":1.073,"15min":1.073},"min":{"1min":0.83499999999999996,"5min":0.83499999999999996,"15min":0.83499999999999996},"max":{"1min":1.4019999999999999,"5min":1.4019999999999999,"15min":1.4019999999999999},"last":0.88400000000000001}]},{"osd":5,"last update":"Mon Jul 15 07:05:19 2024","interfaces":[{"interface":"back","average":{"1min":1.0369999999999999,"5min":0.97699999999999998,"15min":0.96699999999999997},"min":{"1min":0.78600000000000003,"5min":0.68300000000000005,"15min":0.68300000000000005},"max":{"1min":1.3340000000000001,"5min":1.3340000000000001,"15min":1.3340000000000001},"last":0.78200000000000003},{"interface":"front","average":{"1min":1.0129999999999999,"5min":1,"15min":0.998},"min":{"1min":0.68500000000000005,"5min":0.61799999999999999,"15min":0.61799999999999999},"max":{"1min":1.534,"5min":1.534,"15min":1.534},"last":1.0529999999999999}]},{"osd":6,"last update":"Mon Jul 15 07:05:19 2024","interfaces":[{"interface":"back","average":{"1min":1.0289999999999999,"5min":0.995,"15min":0.98899999999999999},"min":{"1min":0.76400000000000001,"5min":0.65200000000000002,"15min":0.65200000000000002},"max":{"1min":1.6060000000000001,"5min":1.6060000000000001,"15min":1.6060000000000001},"last":0.98399999999999999},{"interface":"front","average":{"1min":1.113,"5min":1.0649999999999999,"15min":1.0569999999999999},"min":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"max":{"1min":1.4830000000000001,"5min":1.4830000000000001,"15min":1.4830000000000001},"last":0.93600000000000005}]},{"osd":7,"last update":"Mon Jul 15 07:05:19 2024","interfaces":[{"interface":"back","average":{"1min":1.155,"5min":1.1060000000000001,"15min":1.0980000000000001},"min":{"1min":0.73199999999999998,"5min":0.67000000000000004,"15min":0.67000000000000004},"max":{"1min":1.573,"5min":1.573,"15min":1.573},"last":0.96699999999999997},{"interface":"front","average":{"1min":1.081,"5min":1.0620000000000001,"15min":1.0589999999999999},"min":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"max":{"1min":1.4099999999999999,"5min":1.4690000000000001,"15min":1.4690000000000001},"last":0.90600000000000003}]}]},{"osd":1,"up_from":88,"seq":377957122101,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28792,"kb_used_data":1260,"kb_used_omap":8,"kb_used_meta":27511,"kb_avail":93716360,"statfs":{"total":95995035648,"available":95965552640,"internally_reserved":0,"allocated":1290240,"data_stored":685623,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8660,"internal_metadata":28171820},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Jul 15 07:05:54 2024","interfaces":[{"interface":"back","average":{"1min":0.64300000000000002,"5min":0.64000000000000001,"15min":0.66700000000000004},"min":{"1min":0.26000000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":1.0309999999999999,"5min":1.131,"15min":1.131},"last":0.76500000000000001},{"interface":"front","average":{"1min":0.60499999999999998,"5min":0.70799999999999996,"15min":0.78200000000000003},"min":{"1min":0.32700000000000001,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":1.0289999999999999,"5min":1.131,"15min":1.131},"last":0.72799999999999998}]},{"osd":2,"last update":"Mon Jul 15 07:05:22 2024","interfaces":[{"interface":"back","average":{"1min":0.63700000000000001,"5min":0.68500000000000005,"15min":0.69299999999999995},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":1.0589999999999999,"5min":1.0589999999999999,"15min":1.0589999999999999},"last":0.68100000000000005},{"interface":"front","average":{"1min":0.73299999999999998,"5min":0.72499999999999998,"15min":0.72299999999999998},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":1.1679999999999999,"5min":1.1679999999999999,"15min":1.1679999999999999},"last":1.1180000000000001}]},{"osd":3,"last update":"Mon Jul 15 07:05:48 2024","interfaces":[{"interface":"back","average":{"1min":0.77600000000000002,"5min":0.88700000000000001,"15min":0.90500000000000003},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":1.2330000000000001,"5min":1.2330000000000001,"15min":1.2330000000000001},"last":0.91100000000000003},{"interface":"front","average":{"1min":0.78700000000000003,"5min":0.88300000000000001,"15min":0.89900000000000002},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":1.1759999999999999,"5min":1.2869999999999999,"15min":1.2869999999999999},"last":0.88100000000000001}]},{"osd":4,"last update":"Mon Jul 15 07:05:18 2024","interfaces":[{"interface":"back","average":{"1min":0.86699999999999999,"5min":0.86699999999999999,"15min":0.86699999999999999},"min":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"max":{"1min":1.262,"5min":1.262,"15min":1.262},"last":0.93799999999999994},{"interface":"front","average":{"1min":1.026,"5min":1.026,"15min":1.026},"min":{"1min":0.81399999999999995,"5min":0.81399999999999995,"15min":0.81399999999999995},"max":{"1min":1.325,"5min":1.325,"15min":1.325},"last":1.359}]},{"osd":5,"last update":"Mon Jul 15 07:05:54 2024","interfaces":[{"interface":"back","average":{"1min":0.90000000000000002,"5min":0.94199999999999995,"15min":0.96599999999999997},"min":{"1min":0.54600000000000004,"5min":0.52800000000000002,"15min":0.52800000000000002},"max":{"1min":1.258,"5min":1.4299999999999999,"15min":1.4299999999999999},"last":0.96099999999999997},{"interface":"front","average":{"1min":0.94899999999999995,"5min":0.995,"15min":1.0529999999999999},"min":{"1min":0.68899999999999995,"5min":0.59899999999999998,"15min":0.59899999999999998},"max":{"1min":1.1850000000000001,"5min":1.403,"15min":1.403},"last":1.149}]},{"osd":6,"last update":"Mon Jul 15 07:05:54 2024","interfaces":[{"interface":"back","average":{"1min":0.95899999999999996,"5min":0.999,"15min":1.0309999999999999},"min":{"1min":0.58399999999999996,"5min":0.55300000000000005,"15min":0.55300000000000005},"max":{"1min":1.357,"5min":1.381,"15min":1.381},"last":1.0569999999999999},{"interface":"front","average":{"1min":1.0009999999999999,"5min":1.0580000000000001,"15min":1.127},"min":{"1min":0.69499999999999995,"5min":0.50800000000000001,"15min":0.50800000000000001},"max":{"1min":1.2130000000000001,"5min":1.486,"15min":1.486},"last":1.0189999999999999}]},{"osd":7,"last update":"Mon Jul 15 07:05:54 2024","interfaces":[{"interface":"back","average":{"1min":1.0669999999999999,"5min":1.119,"15min":1.1970000000000001},"min":{"1min":0.72599999999999998,"5min":0.61899999999999999,"15min":0.61899999999999999},"max":{"1min":1.331,"5min":1.702,"15min":1.702},"last":1.304},{"interface":"front","average":{"1min":1.014,"5min":1.097,"15min":1.1519999999999999},"min":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"max":{"1min":1.3109999999999999,"5min":1.4239999999999999,"15min":1.4239999999999999},"last":1.1819999999999999}]}]},{"osd":0,"up_from":83,"seq":356482285628,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":33480,"kb_used_data":1852,"kb_used_omap":18,"kb_used_meta":31597,"kb_avail":93711672,"statfs":{"total":95995035648,"available":95960752128,"internally_reserved":0,"allocated":1896448,"data_stored":1277743,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19384,"internal_metadata":32355400},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Jul 15 07:05:55 2024","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":0.66800000000000004,"15min":0.69099999999999995},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.87,"5min":1.216,"15min":1.216},"last":0.69299999999999995},{"interface":"front","average":{"1min":0.67700000000000005,"5min":0.69499999999999995,"15min":0.70499999999999996},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.871,"5min":1.411,"15min":1.411},"last":1.2529999999999999}]},{"osd":2,"last update":"Mon Jul 15 07:05:21 2024","interfaces":[{"interface":"back","average":{"1min":0.73399999999999999,"5min":0.73699999999999999,"15min":0.73699999999999999},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":1.0329999999999999,"5min":1.0580000000000001,"15min":1.0580000000000001},"last":0.65200000000000002},{"interface":"front","average":{"1min":0.72699999999999998,"5min":0.76600000000000001,"15min":0.77200000000000002},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":1.333,"5min":1.333,"15min":1.333},"last":0.90800000000000003}]},{"osd":3,"last update":"Mon Jul 15 07:05:49 2024","interfaces":[{"interface":"back","average":{"1min":0.69199999999999995,"5min":0.79000000000000004,"15min":0.80600000000000005},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.92300000000000004,"5min":1.1359999999999999,"15min":1.1359999999999999},"last":0.79200000000000004},{"interface":"front","average":{"1min":0.747,"5min":0.76200000000000001,"15min":0.76400000000000001},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":1.0429999999999999,"5min":1.526,"15min":1.526},"last":0.83699999999999997}]},{"osd":4,"last update":"Mon Jul 15 07:05:17 2024","interfaces":[{"interface":"back","average":{"1min":1.002,"5min":1.002,"15min":1.002},"min":{"1min":0.751,"5min":0.751,"15min":0.751},"max":{"1min":1.3080000000000001,"5min":1.3080000000000001,"15min":1.3080000000000001},"last":1.0609999999999999},{"interface":"front","average":{"1min":1.038,"5min":1.038,"15min":1.038},"min":{"1min":0.85199999999999998,"5min":0.85199999999999998,"15min":0.85199999999999998},"max":{"1min":1.4410000000000001,"5min":1.4410000000000001,"15min":1.4410000000000001},"last":0.96099999999999997}]},{"osd":5,"last update":"Mon Jul 15 07:05:21 2024","interfaces":[{"interface":"back","average":{"1min":0.996,"5min":1.0069999999999999,"15min":1.0409999999999999},"min":{"1min":0.752,"5min":0.62,"15min":0.62},"max":{"1min":1.2529999999999999,"5min":1.49,"15min":1.49},"last":1.012},{"interface":"front","average":{"1min":1.0680000000000001,"5min":1.0289999999999999,"15min":1.0469999999999999},"min":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"max":{"1min":1.4179999999999999,"5min":1.4179999999999999,"15min":1.4179999999999999},"last":1.202}]},{"osd":6,"last update":"Mon Jul 15 07:05:21 2024","interfaces":[{"interface":"back","average":{"1min":1.083,"5min":1.0760000000000001,"15min":1.0960000000000001},"min":{"1min":0.871,"5min":0.65100000000000002,"15min":0.65100000000000002},"max":{"1min":1.2809999999999999,"5min":1.5820000000000001,"15min":1.5820000000000001},"last":1.121},{"interface":"front","average":{"1min":1.0509999999999999,"5min":1.0620000000000001,"15min":1.089},"min":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"max":{"1min":1.498,"5min":1.5820000000000001,"15min":1.5820000000000001},"last":1.0389999999999999}]},{"osd":7,"last update":"Mon Jul 15 07:05:21 2024","interfaces":[{"interface":"back","average":{"1min":1.1120000000000001,"5min":1.0900000000000001,"15min":1.107},"min":{"1min":0.68400000000000005,"5min":0.66200000000000003,"15min":0.66200000000000003},"max":{"1min":1.47,"5min":1.649,"15min":1.649},"last":1.1479999999999999},{"interface":"front","average":{"1min":1.0409999999999999,"5min":1.077,"15min":1.113},"min":{"1min":0.60199999999999998,"5min":0.59599999999999997,"15min":0.59599999999999997},"max":{"1min":1.2450000000000001,"5min":1.605,"15min":1.605},"last":1.087}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1051,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1336,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-07-15T07:06:07.008 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-07-15T07:06:07.008 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-07-15T07:06:07.008 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-07-15T07:06:07.009 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph health --format=json 2024-07-15T07:06:07.055 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:06:06 smithi134 bash[67624]: cluster 2024-07-15T07:06:04.916350+0000 mgr.y (mgr.25244) 362 : cluster [DBG] pgmap v308: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:06:08.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:06:07 smithi134 bash[67624]: audit 2024-07-15T07:06:06.237534+0000 mgr.y (mgr.25244) 363 : audit [DBG] from='client.44432 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-07-15T07:06:08.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:06:07 smithi089 bash[79810]: audit 2024-07-15T07:06:06.237534+0000 mgr.y (mgr.25244) 363 : audit [DBG] from='client.44432 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-07-15T07:06:08.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:06:07 smithi089 bash[82202]: audit 2024-07-15T07:06:06.237534+0000 mgr.y (mgr.25244) 363 : audit [DBG] from='client.44432 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-07-15T07:06:09.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:06:08 smithi134 bash[67624]: cluster 2024-07-15T07:06:06.917432+0000 mgr.y (mgr.25244) 364 : cluster [DBG] pgmap v309: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:06:09.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:06:08 smithi089 bash[79810]: cluster 2024-07-15T07:06:06.917432+0000 mgr.y (mgr.25244) 364 : cluster [DBG] pgmap v309: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:06:09.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:06:08 smithi089 bash[82202]: cluster 2024-07-15T07:06:06.917432+0000 mgr.y (mgr.25244) 364 : cluster [DBG] pgmap v309: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:06:11.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:06:10 smithi134 bash[67624]: cluster 2024-07-15T07:06:08.918330+0000 mgr.y (mgr.25244) 365 : cluster [DBG] pgmap v310: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:06:11.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:06:10 smithi089 bash[79810]: cluster 2024-07-15T07:06:08.918330+0000 mgr.y (mgr.25244) 365 : cluster [DBG] pgmap v310: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:06:11.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:06:10 smithi089 bash[82202]: cluster 2024-07-15T07:06:08.918330+0000 mgr.y (mgr.25244) 365 : cluster [DBG] pgmap v310: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:06:11.782 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T07:06:12.657 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:06:12 smithi134 bash[60729]: ts=2024-07-15T07:06:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T07:06:12.956 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:06:12 smithi089 bash[79810]: cluster 2024-07-15T07:06:10.919342+0000 mgr.y (mgr.25244) 366 : cluster [DBG] pgmap v311: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:06:12.956 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:06:12 smithi089 bash[82202]: cluster 2024-07-15T07:06:10.919342+0000 mgr.y (mgr.25244) 366 : cluster [DBG] pgmap v311: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:06:13.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:06:12 smithi134 bash[67624]: cluster 2024-07-15T07:06:10.919342+0000 mgr.y (mgr.25244) 366 : cluster [DBG] pgmap v311: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:06:14.372 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 07:06:14 smithi089 bash[72894]: ::ffff:172.21.15.134 - - [15/Jul/2024:07:06:14] "GET /metrics HTTP/1.1" 200 33979 "" "Prometheus/2.43.0" 2024-07-15T07:06:15.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:06:14 smithi134 bash[67624]: cluster 2024-07-15T07:06:12.919997+0000 mgr.y (mgr.25244) 367 : cluster [DBG] pgmap v312: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:06:15.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:06:14 smithi089 bash[79810]: cluster 2024-07-15T07:06:12.919997+0000 mgr.y (mgr.25244) 367 : cluster [DBG] pgmap v312: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:06:15.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:06:14 smithi089 bash[82202]: cluster 2024-07-15T07:06:12.919997+0000 mgr.y (mgr.25244) 367 : cluster [DBG] pgmap v312: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:06:15.477 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-07-15T07:06:15.477 INFO:teuthology.orchestra.run.smithi089.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-07-15T07:06:15.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:06:15 smithi089 bash[79810]: audit 2024-07-15T07:06:15.480378+0000 mon.c (mon.2) 21 : audit [DBG] from='client.? 172.21.15.89:0/2621666615' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-15T07:06:15.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:06:15 smithi089 bash[82202]: audit 2024-07-15T07:06:15.480378+0000 mon.c (mon.2) 21 : audit [DBG] from='client.? 172.21.15.89:0/2621666615' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-15T07:06:16.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:06:15 smithi134 bash[67624]: audit 2024-07-15T07:06:15.480378+0000 mon.c (mon.2) 21 : audit [DBG] from='client.? 172.21.15.89:0/2621666615' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-15T07:06:17.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:06:16 smithi134 bash[67624]: cluster 2024-07-15T07:06:14.920799+0000 mgr.y (mgr.25244) 368 : cluster [DBG] pgmap v313: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:06:17.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:06:16 smithi089 bash[79810]: cluster 2024-07-15T07:06:14.920799+0000 mgr.y (mgr.25244) 368 : cluster [DBG] pgmap v313: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:06:17.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:06:16 smithi089 bash[82202]: cluster 2024-07-15T07:06:14.920799+0000 mgr.y (mgr.25244) 368 : cluster [DBG] pgmap v313: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:06:19.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:06:18 smithi134 bash[67624]: cluster 2024-07-15T07:06:16.921917+0000 mgr.y (mgr.25244) 369 : cluster [DBG] pgmap v314: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:06:19.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:06:18 smithi089 bash[79810]: cluster 2024-07-15T07:06:16.921917+0000 mgr.y (mgr.25244) 369 : cluster [DBG] pgmap v314: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:06:19.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:06:18 smithi089 bash[82202]: cluster 2024-07-15T07:06:16.921917+0000 mgr.y (mgr.25244) 369 : cluster [DBG] pgmap v314: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:06:19.223 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph health --format=json 2024-07-15T07:06:21.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:06:20 smithi134 bash[67624]: cluster 2024-07-15T07:06:18.922639+0000 mgr.y (mgr.25244) 370 : cluster [DBG] pgmap v315: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:06:21.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:06:20 smithi134 bash[67624]: audit 2024-07-15T07:06:19.797623+0000 mon.a (mon.0) 603 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:06:21.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:06:20 smithi089 bash[79810]: cluster 2024-07-15T07:06:18.922639+0000 mgr.y (mgr.25244) 370 : cluster [DBG] pgmap v315: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:06:21.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:06:20 smithi089 bash[79810]: audit 2024-07-15T07:06:19.797623+0000 mon.a (mon.0) 603 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:06:21.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:06:20 smithi089 bash[82202]: cluster 2024-07-15T07:06:18.922639+0000 mgr.y (mgr.25244) 370 : cluster [DBG] pgmap v315: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:06:21.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:06:20 smithi089 bash[82202]: audit 2024-07-15T07:06:19.797623+0000 mon.a (mon.0) 603 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:06:22.685 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:06:22 smithi134 bash[60729]: ts=2024-07-15T07:06:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T07:06:23.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:06:22 smithi134 bash[67624]: cluster 2024-07-15T07:06:20.923613+0000 mgr.y (mgr.25244) 371 : cluster [DBG] pgmap v316: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:06:23.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:06:22 smithi089 bash[79810]: cluster 2024-07-15T07:06:20.923613+0000 mgr.y (mgr.25244) 371 : cluster [DBG] pgmap v316: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:06:23.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:06:22 smithi089 bash[82202]: cluster 2024-07-15T07:06:20.923613+0000 mgr.y (mgr.25244) 371 : cluster [DBG] pgmap v316: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:06:24.009 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T07:06:24.372 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 07:06:24 smithi089 bash[72894]: ::ffff:172.21.15.134 - - [15/Jul/2024:07:06:24] "GET /metrics HTTP/1.1" 200 33977 "" "Prometheus/2.43.0" 2024-07-15T07:06:25.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:06:24 smithi134 bash[67624]: cluster 2024-07-15T07:06:22.924304+0000 mgr.y (mgr.25244) 372 : cluster [DBG] pgmap v317: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:06:25.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:06:24 smithi089 bash[79810]: cluster 2024-07-15T07:06:22.924304+0000 mgr.y (mgr.25244) 372 : cluster [DBG] pgmap v317: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:06:25.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:06:24 smithi089 bash[82202]: cluster 2024-07-15T07:06:22.924304+0000 mgr.y (mgr.25244) 372 : cluster [DBG] pgmap v317: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:06:27.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:06:26 smithi134 bash[67624]: cluster 2024-07-15T07:06:24.924829+0000 mgr.y (mgr.25244) 373 : cluster [DBG] pgmap v318: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:06:27.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:06:26 smithi089 bash[79810]: cluster 2024-07-15T07:06:24.924829+0000 mgr.y (mgr.25244) 373 : cluster [DBG] pgmap v318: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:06:27.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:06:26 smithi089 bash[82202]: cluster 2024-07-15T07:06:24.924829+0000 mgr.y (mgr.25244) 373 : cluster [DBG] pgmap v318: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:06:27.584 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-07-15T07:06:27.585 INFO:teuthology.orchestra.run.smithi089.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-07-15T07:06:27.869 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:06:27 smithi089 bash[82202]: audit 2024-07-15T07:06:27.587682+0000 mon.c (mon.2) 22 : audit [DBG] from='client.? 172.21.15.89:0/3960213141' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-15T07:06:27.869 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:06:27 smithi089 bash[79810]: audit 2024-07-15T07:06:27.587682+0000 mon.c (mon.2) 22 : audit [DBG] from='client.? 172.21.15.89:0/3960213141' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-15T07:06:28.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:06:27 smithi134 bash[67624]: audit 2024-07-15T07:06:27.587682+0000 mon.c (mon.2) 22 : audit [DBG] from='client.? 172.21.15.89:0/3960213141' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-15T07:06:29.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:06:28 smithi134 bash[67624]: cluster 2024-07-15T07:06:26.925770+0000 mgr.y (mgr.25244) 374 : cluster [DBG] pgmap v319: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:06:29.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:06:28 smithi089 bash[79810]: cluster 2024-07-15T07:06:26.925770+0000 mgr.y (mgr.25244) 374 : cluster [DBG] pgmap v319: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:06:29.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:06:28 smithi089 bash[82202]: cluster 2024-07-15T07:06:26.925770+0000 mgr.y (mgr.25244) 374 : cluster [DBG] pgmap v319: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:06:31.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:06:30 smithi134 bash[67624]: cluster 2024-07-15T07:06:28.926515+0000 mgr.y (mgr.25244) 375 : cluster [DBG] pgmap v320: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:06:31.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:06:30 smithi089 bash[79810]: cluster 2024-07-15T07:06:28.926515+0000 mgr.y (mgr.25244) 375 : cluster [DBG] pgmap v320: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:06:31.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:06:30 smithi089 bash[82202]: cluster 2024-07-15T07:06:28.926515+0000 mgr.y (mgr.25244) 375 : cluster [DBG] pgmap v320: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:06:31.376 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph health --format=json 2024-07-15T07:06:32.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:06:31 smithi134 bash[67624]: audit 2024-07-15T07:06:30.985082+0000 mon.a (mon.0) 604 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T07:06:32.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:06:31 smithi089 bash[79810]: audit 2024-07-15T07:06:30.985082+0000 mon.a (mon.0) 604 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T07:06:32.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:06:31 smithi089 bash[82202]: audit 2024-07-15T07:06:30.985082+0000 mon.a (mon.0) 604 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T07:06:32.717 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:06:32 smithi134 bash[60729]: ts=2024-07-15T07:06:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T07:06:33.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:06:32 smithi134 bash[67624]: cluster 2024-07-15T07:06:30.927470+0000 mgr.y (mgr.25244) 376 : cluster [DBG] pgmap v321: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:06:33.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:06:32 smithi089 bash[79810]: cluster 2024-07-15T07:06:30.927470+0000 mgr.y (mgr.25244) 376 : cluster [DBG] pgmap v321: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:06:33.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:06:32 smithi089 bash[82202]: cluster 2024-07-15T07:06:30.927470+0000 mgr.y (mgr.25244) 376 : cluster [DBG] pgmap v321: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:06:34.622 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 07:06:34 smithi089 bash[72894]: ::ffff:172.21.15.134 - - [15/Jul/2024:07:06:34] "GET /metrics HTTP/1.1" 200 33977 "" "Prometheus/2.43.0" 2024-07-15T07:06:35.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:06:34 smithi134 bash[67624]: cluster 2024-07-15T07:06:32.928162+0000 mgr.y (mgr.25244) 377 : cluster [DBG] pgmap v322: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:06:35.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:06:34 smithi089 bash[79810]: cluster 2024-07-15T07:06:32.928162+0000 mgr.y (mgr.25244) 377 : cluster [DBG] pgmap v322: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:06:35.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:06:34 smithi089 bash[82202]: cluster 2024-07-15T07:06:32.928162+0000 mgr.y (mgr.25244) 377 : cluster [DBG] pgmap v322: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:06:35.898 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T07:06:35.989 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:06:35 smithi089 bash[79810]: audit 2024-07-15T07:06:34.798016+0000 mon.a (mon.0) 605 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:06:35.990 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:06:35 smithi089 bash[82202]: audit 2024-07-15T07:06:34.798016+0000 mon.a (mon.0) 605 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:06:36.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:06:35 smithi134 bash[67624]: audit 2024-07-15T07:06:34.798016+0000 mon.a (mon.0) 605 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:06:37.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:06:36 smithi134 bash[67624]: cluster 2024-07-15T07:06:34.928926+0000 mgr.y (mgr.25244) 378 : cluster [DBG] pgmap v323: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:06:37.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:06:36 smithi134 bash[67624]: audit 2024-07-15T07:06:36.580572+0000 mon.a (mon.0) 606 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:06:37.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:06:36 smithi134 bash[67624]: audit 2024-07-15T07:06:36.588231+0000 mon.a (mon.0) 607 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:06:37.109 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:06:36 smithi089 bash[82202]: cluster 2024-07-15T07:06:34.928926+0000 mgr.y (mgr.25244) 378 : cluster [DBG] pgmap v323: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:06:37.109 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:06:36 smithi089 bash[82202]: audit 2024-07-15T07:06:36.580572+0000 mon.a (mon.0) 606 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:06:37.109 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:06:36 smithi089 bash[82202]: audit 2024-07-15T07:06:36.588231+0000 mon.a (mon.0) 607 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:06:37.109 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:06:36 smithi089 bash[79810]: cluster 2024-07-15T07:06:34.928926+0000 mgr.y (mgr.25244) 378 : cluster [DBG] pgmap v323: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:06:37.109 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:06:36 smithi089 bash[79810]: audit 2024-07-15T07:06:36.580572+0000 mon.a (mon.0) 606 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:06:37.109 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:06:36 smithi089 bash[79810]: audit 2024-07-15T07:06:36.588231+0000 mon.a (mon.0) 607 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:06:38.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:06:37 smithi134 bash[67624]: audit 2024-07-15T07:06:36.794743+0000 mon.a (mon.0) 608 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:06:38.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:06:37 smithi134 bash[67624]: audit 2024-07-15T07:06:36.806550+0000 mon.a (mon.0) 609 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:06:38.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:06:37 smithi134 bash[67624]: audit 2024-07-15T07:06:37.229640+0000 mon.a (mon.0) 610 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T07:06:38.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:06:37 smithi134 bash[67624]: audit 2024-07-15T07:06:37.230406+0000 mon.a (mon.0) 611 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T07:06:38.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:06:37 smithi134 bash[67624]: audit 2024-07-15T07:06:37.235481+0000 mon.a (mon.0) 612 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:06:38.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:06:37 smithi089 bash[79810]: audit 2024-07-15T07:06:36.794743+0000 mon.a (mon.0) 608 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:06:38.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:06:37 smithi089 bash[79810]: audit 2024-07-15T07:06:36.806550+0000 mon.a (mon.0) 609 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:06:38.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:06:37 smithi089 bash[79810]: audit 2024-07-15T07:06:37.229640+0000 mon.a (mon.0) 610 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T07:06:38.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:06:37 smithi089 bash[79810]: audit 2024-07-15T07:06:37.230406+0000 mon.a (mon.0) 611 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T07:06:38.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:06:37 smithi089 bash[79810]: audit 2024-07-15T07:06:37.235481+0000 mon.a (mon.0) 612 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:06:38.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:06:37 smithi089 bash[82202]: audit 2024-07-15T07:06:36.794743+0000 mon.a (mon.0) 608 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:06:38.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:06:37 smithi089 bash[82202]: audit 2024-07-15T07:06:36.806550+0000 mon.a (mon.0) 609 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:06:38.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:06:37 smithi089 bash[82202]: audit 2024-07-15T07:06:37.229640+0000 mon.a (mon.0) 610 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T07:06:38.123 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:06:37 smithi089 bash[82202]: audit 2024-07-15T07:06:37.230406+0000 mon.a (mon.0) 611 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T07:06:38.123 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:06:37 smithi089 bash[82202]: audit 2024-07-15T07:06:37.235481+0000 mon.a (mon.0) 612 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:06:39.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:06:38 smithi089 bash[79810]: cluster 2024-07-15T07:06:36.929658+0000 mgr.y (mgr.25244) 379 : cluster [DBG] pgmap v324: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:06:39.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:06:38 smithi089 bash[82202]: cluster 2024-07-15T07:06:36.929658+0000 mgr.y (mgr.25244) 379 : cluster [DBG] pgmap v324: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:06:39.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:06:38 smithi134 bash[67624]: cluster 2024-07-15T07:06:36.929658+0000 mgr.y (mgr.25244) 379 : cluster [DBG] pgmap v324: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:06:39.732 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-07-15T07:06:39.733 INFO:teuthology.orchestra.run.smithi089.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-07-15T07:06:40.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:06:39 smithi089 bash[79810]: audit 2024-07-15T07:06:39.735566+0000 mon.a (mon.0) 613 : audit [DBG] from='client.? 172.21.15.89:0/2946748292' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-15T07:06:40.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:06:39 smithi089 bash[82202]: audit 2024-07-15T07:06:39.735566+0000 mon.a (mon.0) 613 : audit [DBG] from='client.? 172.21.15.89:0/2946748292' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-15T07:06:40.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:06:39 smithi134 bash[67624]: audit 2024-07-15T07:06:39.735566+0000 mon.a (mon.0) 613 : audit [DBG] from='client.? 172.21.15.89:0/2946748292' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-15T07:06:41.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:06:40 smithi089 bash[79810]: cluster 2024-07-15T07:06:38.930342+0000 mgr.y (mgr.25244) 380 : cluster [DBG] pgmap v325: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:06:41.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:06:40 smithi089 bash[82202]: cluster 2024-07-15T07:06:38.930342+0000 mgr.y (mgr.25244) 380 : cluster [DBG] pgmap v325: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:06:41.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:06:40 smithi134 bash[67624]: cluster 2024-07-15T07:06:38.930342+0000 mgr.y (mgr.25244) 380 : cluster [DBG] pgmap v325: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:06:42.806 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:06:42 smithi134 bash[60729]: ts=2024-07-15T07:06:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T07:06:43.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:06:42 smithi089 bash[79810]: cluster 2024-07-15T07:06:40.931388+0000 mgr.y (mgr.25244) 381 : cluster [DBG] pgmap v326: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:06:43.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:06:42 smithi089 bash[82202]: cluster 2024-07-15T07:06:40.931388+0000 mgr.y (mgr.25244) 381 : cluster [DBG] pgmap v326: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:06:43.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:06:42 smithi134 bash[67624]: cluster 2024-07-15T07:06:40.931388+0000 mgr.y (mgr.25244) 381 : cluster [DBG] pgmap v326: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:06:43.480 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph health --format=json 2024-07-15T07:06:44.622 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 07:06:44 smithi089 bash[72894]: ::ffff:172.21.15.134 - - [15/Jul/2024:07:06:44] "GET /metrics HTTP/1.1" 200 33973 "" "Prometheus/2.43.0" 2024-07-15T07:06:45.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:06:44 smithi089 bash[79810]: cluster 2024-07-15T07:06:42.932073+0000 mgr.y (mgr.25244) 382 : cluster [DBG] pgmap v327: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:06:45.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:06:44 smithi089 bash[82202]: cluster 2024-07-15T07:06:42.932073+0000 mgr.y (mgr.25244) 382 : cluster [DBG] pgmap v327: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:06:45.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:06:44 smithi134 bash[67624]: cluster 2024-07-15T07:06:42.932073+0000 mgr.y (mgr.25244) 382 : cluster [DBG] pgmap v327: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:06:47.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:06:46 smithi089 bash[79810]: cluster 2024-07-15T07:06:44.932867+0000 mgr.y (mgr.25244) 383 : cluster [DBG] pgmap v328: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:06:47.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:06:46 smithi089 bash[82202]: cluster 2024-07-15T07:06:44.932867+0000 mgr.y (mgr.25244) 383 : cluster [DBG] pgmap v328: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:06:47.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:06:46 smithi134 bash[67624]: cluster 2024-07-15T07:06:44.932867+0000 mgr.y (mgr.25244) 383 : cluster [DBG] pgmap v328: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:06:48.273 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T07:06:49.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:06:48 smithi089 bash[79810]: cluster 2024-07-15T07:06:46.933807+0000 mgr.y (mgr.25244) 384 : cluster [DBG] pgmap v329: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:06:49.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:06:48 smithi089 bash[82202]: cluster 2024-07-15T07:06:46.933807+0000 mgr.y (mgr.25244) 384 : cluster [DBG] pgmap v329: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:06:49.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:06:48 smithi134 bash[67624]: cluster 2024-07-15T07:06:46.933807+0000 mgr.y (mgr.25244) 384 : cluster [DBG] pgmap v329: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:06:50.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:06:49 smithi089 bash[79810]: audit 2024-07-15T07:06:49.797707+0000 mon.a (mon.0) 614 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:06:50.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:06:49 smithi089 bash[82202]: audit 2024-07-15T07:06:49.797707+0000 mon.a (mon.0) 614 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:06:50.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:06:49 smithi134 bash[67624]: audit 2024-07-15T07:06:49.797707+0000 mon.a (mon.0) 614 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:06:51.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:06:50 smithi089 bash[79810]: cluster 2024-07-15T07:06:48.934505+0000 mgr.y (mgr.25244) 385 : cluster [DBG] pgmap v330: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:06:51.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:06:50 smithi089 bash[82202]: cluster 2024-07-15T07:06:48.934505+0000 mgr.y (mgr.25244) 385 : cluster [DBG] pgmap v330: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:06:51.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:06:50 smithi134 bash[67624]: cluster 2024-07-15T07:06:48.934505+0000 mgr.y (mgr.25244) 385 : cluster [DBG] pgmap v330: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:06:52.062 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-07-15T07:06:52.062 INFO:teuthology.orchestra.run.smithi089.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-07-15T07:06:52.806 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:06:52 smithi134 bash[60729]: ts=2024-07-15T07:06:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T07:06:53.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:06:52 smithi089 bash[79810]: cluster 2024-07-15T07:06:50.935584+0000 mgr.y (mgr.25244) 386 : cluster [DBG] pgmap v331: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:06:53.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:06:52 smithi089 bash[79810]: audit 2024-07-15T07:06:52.064847+0000 mon.c (mon.2) 23 : audit [DBG] from='client.? 172.21.15.89:0/965068000' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-15T07:06:53.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:06:52 smithi089 bash[82202]: cluster 2024-07-15T07:06:50.935584+0000 mgr.y (mgr.25244) 386 : cluster [DBG] pgmap v331: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:06:53.123 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:06:52 smithi089 bash[82202]: audit 2024-07-15T07:06:52.064847+0000 mon.c (mon.2) 23 : audit [DBG] from='client.? 172.21.15.89:0/965068000' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-15T07:06:53.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:06:52 smithi134 bash[67624]: cluster 2024-07-15T07:06:50.935584+0000 mgr.y (mgr.25244) 386 : cluster [DBG] pgmap v331: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:06:53.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:06:52 smithi134 bash[67624]: audit 2024-07-15T07:06:52.064847+0000 mon.c (mon.2) 23 : audit [DBG] from='client.? 172.21.15.89:0/965068000' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-15T07:06:54.622 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 07:06:54 smithi089 bash[72894]: ::ffff:172.21.15.134 - - [15/Jul/2024:07:06:54] "GET /metrics HTTP/1.1" 200 33977 "" "Prometheus/2.43.0" 2024-07-15T07:06:55.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:06:54 smithi089 bash[79810]: cluster 2024-07-15T07:06:52.936310+0000 mgr.y (mgr.25244) 387 : cluster [DBG] pgmap v332: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:06:55.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:06:54 smithi089 bash[82202]: cluster 2024-07-15T07:06:52.936310+0000 mgr.y (mgr.25244) 387 : cluster [DBG] pgmap v332: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:06:55.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:06:54 smithi134 bash[67624]: cluster 2024-07-15T07:06:52.936310+0000 mgr.y (mgr.25244) 387 : cluster [DBG] pgmap v332: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:06:55.825 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph health --format=json 2024-07-15T07:06:57.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:06:56 smithi089 bash[79810]: cluster 2024-07-15T07:06:54.937091+0000 mgr.y (mgr.25244) 388 : cluster [DBG] pgmap v333: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:06:57.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:06:56 smithi089 bash[82202]: cluster 2024-07-15T07:06:54.937091+0000 mgr.y (mgr.25244) 388 : cluster [DBG] pgmap v333: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:06:57.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:06:56 smithi134 bash[67624]: cluster 2024-07-15T07:06:54.937091+0000 mgr.y (mgr.25244) 388 : cluster [DBG] pgmap v333: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:06:59.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:06:58 smithi089 bash[79810]: cluster 2024-07-15T07:06:56.937996+0000 mgr.y (mgr.25244) 389 : cluster [DBG] pgmap v334: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:06:59.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:06:58 smithi089 bash[82202]: cluster 2024-07-15T07:06:56.937996+0000 mgr.y (mgr.25244) 389 : cluster [DBG] pgmap v334: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:06:59.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:06:58 smithi134 bash[67624]: cluster 2024-07-15T07:06:56.937996+0000 mgr.y (mgr.25244) 389 : cluster [DBG] pgmap v334: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:07:00.606 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T07:07:01.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:07:00 smithi089 bash[79810]: cluster 2024-07-15T07:06:58.938713+0000 mgr.y (mgr.25244) 390 : cluster [DBG] pgmap v335: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:07:01.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:07:00 smithi089 bash[82202]: cluster 2024-07-15T07:06:58.938713+0000 mgr.y (mgr.25244) 390 : cluster [DBG] pgmap v335: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:07:01.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:07:00 smithi134 bash[67624]: cluster 2024-07-15T07:06:58.938713+0000 mgr.y (mgr.25244) 390 : cluster [DBG] pgmap v335: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:07:02.806 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:07:02 smithi134 bash[60729]: ts=2024-07-15T07:07:02.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T07:07:03.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:07:02 smithi089 bash[79810]: cluster 2024-07-15T07:07:00.939708+0000 mgr.y (mgr.25244) 391 : cluster [DBG] pgmap v336: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:07:03.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:07:02 smithi089 bash[82202]: cluster 2024-07-15T07:07:00.939708+0000 mgr.y (mgr.25244) 391 : cluster [DBG] pgmap v336: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:07:03.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:07:02 smithi134 bash[67624]: cluster 2024-07-15T07:07:00.939708+0000 mgr.y (mgr.25244) 391 : cluster [DBG] pgmap v336: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:07:04.372 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 07:07:04 smithi089 bash[72894]: ::ffff:172.21.15.134 - - [15/Jul/2024:07:07:04] "GET /metrics HTTP/1.1" 200 33977 "" "Prometheus/2.43.0" 2024-07-15T07:07:04.411 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-07-15T07:07:04.411 INFO:teuthology.orchestra.run.smithi089.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-07-15T07:07:05.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:07:04 smithi089 bash[79810]: cluster 2024-07-15T07:07:02.940435+0000 mgr.y (mgr.25244) 392 : cluster [DBG] pgmap v337: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:07:05.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:07:04 smithi089 bash[79810]: audit 2024-07-15T07:07:04.414325+0000 mon.a (mon.0) 615 : audit [DBG] from='client.? 172.21.15.89:0/24024408' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-15T07:07:05.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:07:04 smithi089 bash[79810]: audit 2024-07-15T07:07:04.798608+0000 mon.a (mon.0) 616 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:07:05.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:07:04 smithi089 bash[82202]: cluster 2024-07-15T07:07:02.940435+0000 mgr.y (mgr.25244) 392 : cluster [DBG] pgmap v337: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:07:05.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:07:04 smithi089 bash[82202]: audit 2024-07-15T07:07:04.414325+0000 mon.a (mon.0) 615 : audit [DBG] from='client.? 172.21.15.89:0/24024408' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-15T07:07:05.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:07:04 smithi089 bash[82202]: audit 2024-07-15T07:07:04.798608+0000 mon.a (mon.0) 616 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:07:05.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:07:04 smithi134 bash[67624]: cluster 2024-07-15T07:07:02.940435+0000 mgr.y (mgr.25244) 392 : cluster [DBG] pgmap v337: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:07:05.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:07:04 smithi134 bash[67624]: audit 2024-07-15T07:07:04.414325+0000 mon.a (mon.0) 615 : audit [DBG] from='client.? 172.21.15.89:0/24024408' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-15T07:07:05.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:07:04 smithi134 bash[67624]: audit 2024-07-15T07:07:04.798608+0000 mon.a (mon.0) 616 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:07:07.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:07:06 smithi134 bash[67624]: cluster 2024-07-15T07:07:04.941167+0000 mgr.y (mgr.25244) 393 : cluster [DBG] pgmap v338: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:07:07.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:07:06 smithi089 bash[79810]: cluster 2024-07-15T07:07:04.941167+0000 mgr.y (mgr.25244) 393 : cluster [DBG] pgmap v338: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:07:07.372 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:07:06 smithi089 bash[82202]: cluster 2024-07-15T07:07:04.941167+0000 mgr.y (mgr.25244) 393 : cluster [DBG] pgmap v338: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:07:08.110 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph health --format=json 2024-07-15T07:07:09.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:07:08 smithi134 bash[67624]: cluster 2024-07-15T07:07:06.942139+0000 mgr.y (mgr.25244) 394 : cluster [DBG] pgmap v339: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:07:09.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:07:08 smithi089 bash[79810]: cluster 2024-07-15T07:07:06.942139+0000 mgr.y (mgr.25244) 394 : cluster [DBG] pgmap v339: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:07:09.372 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:07:08 smithi089 bash[82202]: cluster 2024-07-15T07:07:06.942139+0000 mgr.y (mgr.25244) 394 : cluster [DBG] pgmap v339: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:07:11.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:07:10 smithi134 bash[67624]: cluster 2024-07-15T07:07:08.942860+0000 mgr.y (mgr.25244) 395 : cluster [DBG] pgmap v340: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:07:11.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:07:10 smithi089 bash[79810]: cluster 2024-07-15T07:07:08.942860+0000 mgr.y (mgr.25244) 395 : cluster [DBG] pgmap v340: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:07:11.372 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:07:10 smithi089 bash[82202]: cluster 2024-07-15T07:07:08.942860+0000 mgr.y (mgr.25244) 395 : cluster [DBG] pgmap v340: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:07:12.806 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:07:12 smithi134 bash[60729]: ts=2024-07-15T07:07:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T07:07:12.890 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T07:07:13.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:07:12 smithi134 bash[67624]: cluster 2024-07-15T07:07:10.943737+0000 mgr.y (mgr.25244) 396 : cluster [DBG] pgmap v341: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:07:13.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:07:12 smithi089 bash[79810]: cluster 2024-07-15T07:07:10.943737+0000 mgr.y (mgr.25244) 396 : cluster [DBG] pgmap v341: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:07:13.372 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:07:12 smithi089 bash[82202]: cluster 2024-07-15T07:07:10.943737+0000 mgr.y (mgr.25244) 396 : cluster [DBG] pgmap v341: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:07:14.161 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:07:13 smithi089 bash[79810]: cluster 2024-07-15T07:07:12.944237+0000 mgr.y (mgr.25244) 397 : cluster [DBG] pgmap v342: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:07:14.161 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:07:13 smithi089 bash[82202]: cluster 2024-07-15T07:07:12.944237+0000 mgr.y (mgr.25244) 397 : cluster [DBG] pgmap v342: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:07:14.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:07:13 smithi134 bash[67624]: cluster 2024-07-15T07:07:12.944237+0000 mgr.y (mgr.25244) 397 : cluster [DBG] pgmap v342: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:07:14.529 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 07:07:14 smithi089 bash[72894]: ::ffff:172.21.15.134 - - [15/Jul/2024:07:07:14] "GET /metrics HTTP/1.1" 200 33978 "" "Prometheus/2.43.0" 2024-07-15T07:07:16.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:07:16 smithi134 bash[67624]: cluster 2024-07-15T07:07:14.944991+0000 mgr.y (mgr.25244) 398 : cluster [DBG] pgmap v343: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:07:16.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:07:16 smithi089 bash[79810]: cluster 2024-07-15T07:07:14.944991+0000 mgr.y (mgr.25244) 398 : cluster [DBG] pgmap v343: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:07:16.372 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:07:16 smithi089 bash[82202]: cluster 2024-07-15T07:07:14.944991+0000 mgr.y (mgr.25244) 398 : cluster [DBG] pgmap v343: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:07:16.546 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-07-15T07:07:16.546 INFO:teuthology.orchestra.run.smithi089.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-07-15T07:07:17.256 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:07:17 smithi089 bash[79810]: audit 2024-07-15T07:07:16.548926+0000 mon.a (mon.0) 617 : audit [DBG] from='client.? 172.21.15.89:0/1625017520' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-15T07:07:17.256 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:07:17 smithi089 bash[82202]: audit 2024-07-15T07:07:16.548926+0000 mon.a (mon.0) 617 : audit [DBG] from='client.? 172.21.15.89:0/1625017520' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-15T07:07:17.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:07:17 smithi134 bash[67624]: audit 2024-07-15T07:07:16.548926+0000 mon.a (mon.0) 617 : audit [DBG] from='client.? 172.21.15.89:0/1625017520' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-15T07:07:18.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:07:18 smithi134 bash[67624]: cluster 2024-07-15T07:07:16.945921+0000 mgr.y (mgr.25244) 399 : cluster [DBG] pgmap v344: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:07:18.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:07:18 smithi089 bash[79810]: cluster 2024-07-15T07:07:16.945921+0000 mgr.y (mgr.25244) 399 : cluster [DBG] pgmap v344: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:07:18.372 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:07:18 smithi089 bash[82202]: cluster 2024-07-15T07:07:16.945921+0000 mgr.y (mgr.25244) 399 : cluster [DBG] pgmap v344: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:07:20.258 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph health --format=json 2024-07-15T07:07:20.270 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:07:20 smithi089 bash[79810]: cluster 2024-07-15T07:07:18.946631+0000 mgr.y (mgr.25244) 400 : cluster [DBG] pgmap v345: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:07:20.270 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:07:20 smithi089 bash[79810]: audit 2024-07-15T07:07:19.799102+0000 mon.a (mon.0) 618 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:07:20.270 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:07:20 smithi089 bash[82202]: cluster 2024-07-15T07:07:18.946631+0000 mgr.y (mgr.25244) 400 : cluster [DBG] pgmap v345: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:07:20.270 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:07:20 smithi089 bash[82202]: audit 2024-07-15T07:07:19.799102+0000 mon.a (mon.0) 618 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:07:20.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:07:20 smithi134 bash[67624]: cluster 2024-07-15T07:07:18.946631+0000 mgr.y (mgr.25244) 400 : cluster [DBG] pgmap v345: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:07:20.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:07:20 smithi134 bash[67624]: audit 2024-07-15T07:07:19.799102+0000 mon.a (mon.0) 618 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:07:22.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:07:22 smithi134 bash[67624]: cluster 2024-07-15T07:07:20.947606+0000 mgr.y (mgr.25244) 401 : cluster [DBG] pgmap v346: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:07:22.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:07:22 smithi089 bash[79810]: cluster 2024-07-15T07:07:20.947606+0000 mgr.y (mgr.25244) 401 : cluster [DBG] pgmap v346: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:07:22.372 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:07:22 smithi089 bash[82202]: cluster 2024-07-15T07:07:20.947606+0000 mgr.y (mgr.25244) 401 : cluster [DBG] pgmap v346: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:07:22.806 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:07:22 smithi134 bash[60729]: ts=2024-07-15T07:07:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T07:07:24.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:07:24 smithi134 bash[67624]: cluster 2024-07-15T07:07:22.948315+0000 mgr.y (mgr.25244) 402 : cluster [DBG] pgmap v347: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:07:24.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:07:24 smithi089 bash[79810]: cluster 2024-07-15T07:07:22.948315+0000 mgr.y (mgr.25244) 402 : cluster [DBG] pgmap v347: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:07:24.372 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 07:07:24 smithi089 bash[72894]: ::ffff:172.21.15.134 - - [15/Jul/2024:07:07:24] "GET /metrics HTTP/1.1" 200 33979 "" "Prometheus/2.43.0" 2024-07-15T07:07:24.372 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:07:24 smithi089 bash[82202]: cluster 2024-07-15T07:07:22.948315+0000 mgr.y (mgr.25244) 402 : cluster [DBG] pgmap v347: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:07:25.051 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T07:07:26.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:07:26 smithi134 bash[67624]: cluster 2024-07-15T07:07:24.949105+0000 mgr.y (mgr.25244) 403 : cluster [DBG] pgmap v348: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:07:26.319 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:07:26 smithi089 bash[79810]: cluster 2024-07-15T07:07:24.949105+0000 mgr.y (mgr.25244) 403 : cluster [DBG] pgmap v348: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:07:26.320 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:07:26 smithi089 bash[82202]: cluster 2024-07-15T07:07:24.949105+0000 mgr.y (mgr.25244) 403 : cluster [DBG] pgmap v348: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:07:28.304 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:07:28 smithi089 bash[79810]: cluster 2024-07-15T07:07:26.950047+0000 mgr.y (mgr.25244) 404 : cluster [DBG] pgmap v349: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:07:28.304 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:07:28 smithi089 bash[82202]: cluster 2024-07-15T07:07:26.950047+0000 mgr.y (mgr.25244) 404 : cluster [DBG] pgmap v349: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:07:28.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:07:28 smithi134 bash[67624]: cluster 2024-07-15T07:07:26.950047+0000 mgr.y (mgr.25244) 404 : cluster [DBG] pgmap v349: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:07:28.678 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-07-15T07:07:28.678 INFO:teuthology.orchestra.run.smithi089.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-07-15T07:07:29.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:07:29 smithi134 bash[67624]: audit 2024-07-15T07:07:28.680673+0000 mon.c (mon.2) 24 : audit [DBG] from='client.? 172.21.15.89:0/1350876173' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-15T07:07:29.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:07:29 smithi089 bash[79810]: audit 2024-07-15T07:07:28.680673+0000 mon.c (mon.2) 24 : audit [DBG] from='client.? 172.21.15.89:0/1350876173' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-15T07:07:29.372 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:07:29 smithi089 bash[82202]: audit 2024-07-15T07:07:28.680673+0000 mon.c (mon.2) 24 : audit [DBG] from='client.? 172.21.15.89:0/1350876173' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-15T07:07:30.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:07:30 smithi134 bash[67624]: cluster 2024-07-15T07:07:28.950790+0000 mgr.y (mgr.25244) 405 : cluster [DBG] pgmap v350: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:07:30.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:07:30 smithi089 bash[79810]: cluster 2024-07-15T07:07:28.950790+0000 mgr.y (mgr.25244) 405 : cluster [DBG] pgmap v350: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:07:30.372 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:07:30 smithi089 bash[82202]: cluster 2024-07-15T07:07:28.950790+0000 mgr.y (mgr.25244) 405 : cluster [DBG] pgmap v350: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:07:32.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:07:32 smithi134 bash[67624]: cluster 2024-07-15T07:07:30.951815+0000 mgr.y (mgr.25244) 406 : cluster [DBG] pgmap v351: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:07:32.374 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:07:32 smithi089 bash[79810]: cluster 2024-07-15T07:07:30.951815+0000 mgr.y (mgr.25244) 406 : cluster [DBG] pgmap v351: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:07:32.374 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:07:32 smithi089 bash[82202]: cluster 2024-07-15T07:07:30.951815+0000 mgr.y (mgr.25244) 406 : cluster [DBG] pgmap v351: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:07:32.403 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph health --format=json 2024-07-15T07:07:32.806 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:07:32 smithi134 bash[60729]: ts=2024-07-15T07:07:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T07:07:34.306 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:07:34 smithi134 bash[67624]: cluster 2024-07-15T07:07:32.952572+0000 mgr.y (mgr.25244) 407 : cluster [DBG] pgmap v352: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:07:34.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:07:34 smithi089 bash[79810]: cluster 2024-07-15T07:07:32.952572+0000 mgr.y (mgr.25244) 407 : cluster [DBG] pgmap v352: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:07:34.372 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 07:07:34 smithi089 bash[72894]: ::ffff:172.21.15.134 - - [15/Jul/2024:07:07:34] "GET /metrics HTTP/1.1" 200 33979 "" "Prometheus/2.43.0" 2024-07-15T07:07:34.372 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:07:34 smithi089 bash[82202]: cluster 2024-07-15T07:07:32.952572+0000 mgr.y (mgr.25244) 407 : cluster [DBG] pgmap v352: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:07:35.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:07:35 smithi089 bash[79810]: audit 2024-07-15T07:07:34.799691+0000 mon.a (mon.0) 619 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:07:35.372 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:07:35 smithi089 bash[82202]: audit 2024-07-15T07:07:34.799691+0000 mon.a (mon.0) 619 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:07:35.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:07:35 smithi134 bash[67624]: audit 2024-07-15T07:07:34.799691+0000 mon.a (mon.0) 619 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:07:36.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:07:36 smithi089 bash[79810]: cluster 2024-07-15T07:07:34.953371+0000 mgr.y (mgr.25244) 408 : cluster [DBG] pgmap v353: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:07:36.372 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:07:36 smithi089 bash[82202]: cluster 2024-07-15T07:07:34.953371+0000 mgr.y (mgr.25244) 408 : cluster [DBG] pgmap v353: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:07:36.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:07:36 smithi134 bash[67624]: cluster 2024-07-15T07:07:34.953371+0000 mgr.y (mgr.25244) 408 : cluster [DBG] pgmap v353: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:07:37.185 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T07:07:38.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:07:38 smithi089 bash[79810]: cluster 2024-07-15T07:07:36.954355+0000 mgr.y (mgr.25244) 409 : cluster [DBG] pgmap v354: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:07:38.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:07:38 smithi089 bash[79810]: audit 2024-07-15T07:07:37.308880+0000 mon.a (mon.0) 620 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T07:07:38.372 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:07:38 smithi089 bash[82202]: cluster 2024-07-15T07:07:36.954355+0000 mgr.y (mgr.25244) 409 : cluster [DBG] pgmap v354: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:07:38.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:07:38 smithi089 bash[82202]: audit 2024-07-15T07:07:37.308880+0000 mon.a (mon.0) 620 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T07:07:38.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:07:38 smithi134 bash[67624]: cluster 2024-07-15T07:07:36.954355+0000 mgr.y (mgr.25244) 409 : cluster [DBG] pgmap v354: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:07:38.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:07:38 smithi134 bash[67624]: audit 2024-07-15T07:07:37.308880+0000 mon.a (mon.0) 620 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T07:07:40.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:07:40 smithi089 bash[79810]: cluster 2024-07-15T07:07:38.955087+0000 mgr.y (mgr.25244) 410 : cluster [DBG] pgmap v355: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:07:40.372 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:07:40 smithi089 bash[82202]: cluster 2024-07-15T07:07:38.955087+0000 mgr.y (mgr.25244) 410 : cluster [DBG] pgmap v355: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:07:40.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:07:40 smithi134 bash[67624]: cluster 2024-07-15T07:07:38.955087+0000 mgr.y (mgr.25244) 410 : cluster [DBG] pgmap v355: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:07:40.870 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-07-15T07:07:40.870 INFO:teuthology.orchestra.run.smithi089.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-07-15T07:07:41.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:07:41 smithi089 bash[79810]: audit 2024-07-15T07:07:40.872770+0000 mon.a (mon.0) 621 : audit [DBG] from='client.? 172.21.15.89:0/1348234088' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-15T07:07:41.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:07:41 smithi089 bash[82202]: audit 2024-07-15T07:07:40.872770+0000 mon.a (mon.0) 621 : audit [DBG] from='client.? 172.21.15.89:0/1348234088' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-15T07:07:41.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:07:41 smithi134 bash[67624]: audit 2024-07-15T07:07:40.872770+0000 mon.a (mon.0) 621 : audit [DBG] from='client.? 172.21.15.89:0/1348234088' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-15T07:07:42.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:07:42 smithi089 bash[79810]: cluster 2024-07-15T07:07:40.955745+0000 mgr.y (mgr.25244) 411 : cluster [DBG] pgmap v356: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:07:42.372 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:07:42 smithi089 bash[82202]: cluster 2024-07-15T07:07:40.955745+0000 mgr.y (mgr.25244) 411 : cluster [DBG] pgmap v356: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:07:42.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:07:42 smithi134 bash[67624]: cluster 2024-07-15T07:07:40.955745+0000 mgr.y (mgr.25244) 411 : cluster [DBG] pgmap v356: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:07:42.556 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:07:42 smithi134 bash[60729]: ts=2024-07-15T07:07:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T07:07:44.372 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 07:07:44 smithi089 bash[72894]: ::ffff:172.21.15.134 - - [15/Jul/2024:07:07:44] "GET /metrics HTTP/1.1" 200 33978 "" "Prometheus/2.43.0" 2024-07-15T07:07:44.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:07:44 smithi089 bash[79810]: cluster 2024-07-15T07:07:42.956224+0000 mgr.y (mgr.25244) 412 : cluster [DBG] pgmap v357: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:07:44.373 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:07:44 smithi089 bash[79810]: audit 2024-07-15T07:07:43.075330+0000 mon.a (mon.0) 622 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:07:44.373 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:07:44 smithi089 bash[79810]: audit 2024-07-15T07:07:43.084117+0000 mon.a (mon.0) 623 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:07:44.373 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:07:44 smithi089 bash[79810]: audit 2024-07-15T07:07:43.141361+0000 mon.a (mon.0) 624 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:07:44.373 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:07:44 smithi089 bash[79810]: audit 2024-07-15T07:07:43.149797+0000 mon.a (mon.0) 625 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:07:44.373 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:07:44 smithi089 bash[79810]: audit 2024-07-15T07:07:43.617770+0000 mon.a (mon.0) 626 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T07:07:44.373 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:07:44 smithi089 bash[79810]: audit 2024-07-15T07:07:43.619064+0000 mon.a (mon.0) 627 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T07:07:44.373 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:07:44 smithi089 bash[79810]: audit 2024-07-15T07:07:43.629141+0000 mon.a (mon.0) 628 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:07:44.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:07:44 smithi089 bash[82202]: cluster 2024-07-15T07:07:42.956224+0000 mgr.y (mgr.25244) 412 : cluster [DBG] pgmap v357: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:07:44.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:07:44 smithi089 bash[82202]: audit 2024-07-15T07:07:43.075330+0000 mon.a (mon.0) 622 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:07:44.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:07:44 smithi089 bash[82202]: audit 2024-07-15T07:07:43.084117+0000 mon.a (mon.0) 623 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:07:44.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:07:44 smithi089 bash[82202]: audit 2024-07-15T07:07:43.141361+0000 mon.a (mon.0) 624 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:07:44.374 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:07:44 smithi089 bash[82202]: audit 2024-07-15T07:07:43.149797+0000 mon.a (mon.0) 625 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:07:44.374 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:07:44 smithi089 bash[82202]: audit 2024-07-15T07:07:43.617770+0000 mon.a (mon.0) 626 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T07:07:44.374 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:07:44 smithi089 bash[82202]: audit 2024-07-15T07:07:43.619064+0000 mon.a (mon.0) 627 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T07:07:44.374 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:07:44 smithi089 bash[82202]: audit 2024-07-15T07:07:43.629141+0000 mon.a (mon.0) 628 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:07:44.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:07:44 smithi134 bash[67624]: cluster 2024-07-15T07:07:42.956224+0000 mgr.y (mgr.25244) 412 : cluster [DBG] pgmap v357: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:07:44.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:07:44 smithi134 bash[67624]: audit 2024-07-15T07:07:43.075330+0000 mon.a (mon.0) 622 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:07:44.557 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:07:44 smithi134 bash[67624]: audit 2024-07-15T07:07:43.084117+0000 mon.a (mon.0) 623 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:07:44.557 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:07:44 smithi134 bash[67624]: audit 2024-07-15T07:07:43.141361+0000 mon.a (mon.0) 624 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:07:44.557 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:07:44 smithi134 bash[67624]: audit 2024-07-15T07:07:43.149797+0000 mon.a (mon.0) 625 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:07:44.557 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:07:44 smithi134 bash[67624]: audit 2024-07-15T07:07:43.617770+0000 mon.a (mon.0) 626 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T07:07:44.557 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:07:44 smithi134 bash[67624]: audit 2024-07-15T07:07:43.619064+0000 mon.a (mon.0) 627 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T07:07:44.557 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:07:44 smithi134 bash[67624]: audit 2024-07-15T07:07:43.629141+0000 mon.a (mon.0) 628 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:07:44.657 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph health --format=json 2024-07-15T07:07:46.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:07:46 smithi089 bash[79810]: cluster 2024-07-15T07:07:44.956985+0000 mgr.y (mgr.25244) 413 : cluster [DBG] pgmap v358: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:07:46.372 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:07:46 smithi089 bash[82202]: cluster 2024-07-15T07:07:44.956985+0000 mgr.y (mgr.25244) 413 : cluster [DBG] pgmap v358: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:07:46.556 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:07:46 smithi134 bash[67624]: cluster 2024-07-15T07:07:44.956985+0000 mgr.y (mgr.25244) 413 : cluster [DBG] pgmap v358: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:07:48.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:07:48 smithi134 bash[67624]: cluster 2024-07-15T07:07:46.957981+0000 mgr.y (mgr.25244) 414 : cluster [DBG] pgmap v359: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:07:48.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:07:48 smithi089 bash[79810]: cluster 2024-07-15T07:07:46.957981+0000 mgr.y (mgr.25244) 414 : cluster [DBG] pgmap v359: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:07:48.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:07:48 smithi089 bash[82202]: cluster 2024-07-15T07:07:46.957981+0000 mgr.y (mgr.25244) 414 : cluster [DBG] pgmap v359: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:07:49.444 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T07:07:50.724 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:07:50 smithi089 bash[79810]: cluster 2024-07-15T07:07:48.958733+0000 mgr.y (mgr.25244) 415 : cluster [DBG] pgmap v360: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:07:50.724 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:07:50 smithi089 bash[79810]: audit 2024-07-15T07:07:49.799968+0000 mon.a (mon.0) 629 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:07:50.724 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:07:50 smithi089 bash[82202]: cluster 2024-07-15T07:07:48.958733+0000 mgr.y (mgr.25244) 415 : cluster [DBG] pgmap v360: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:07:50.724 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:07:50 smithi089 bash[82202]: audit 2024-07-15T07:07:49.799968+0000 mon.a (mon.0) 629 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:07:50.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:07:50 smithi134 bash[67624]: cluster 2024-07-15T07:07:48.958733+0000 mgr.y (mgr.25244) 415 : cluster [DBG] pgmap v360: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:07:50.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:07:50 smithi134 bash[67624]: audit 2024-07-15T07:07:49.799968+0000 mon.a (mon.0) 629 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:07:52.775 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:07:52 smithi089 bash[79810]: cluster 2024-07-15T07:07:50.959750+0000 mgr.y (mgr.25244) 416 : cluster [DBG] pgmap v361: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:07:52.775 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:07:52 smithi089 bash[82202]: cluster 2024-07-15T07:07:50.959750+0000 mgr.y (mgr.25244) 416 : cluster [DBG] pgmap v361: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:07:52.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:07:52 smithi134 bash[67624]: cluster 2024-07-15T07:07:50.959750+0000 mgr.y (mgr.25244) 416 : cluster [DBG] pgmap v361: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:07:52.806 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:07:52 smithi134 bash[60729]: ts=2024-07-15T07:07:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T07:07:53.152 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-07-15T07:07:53.152 INFO:teuthology.orchestra.run.smithi089.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-07-15T07:07:53.622 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:07:53 smithi089 bash[79810]: audit 2024-07-15T07:07:53.155239+0000 mon.a (mon.0) 630 : audit [DBG] from='client.? 172.21.15.89:0/1781617151' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-15T07:07:53.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:07:53 smithi089 bash[82202]: audit 2024-07-15T07:07:53.155239+0000 mon.a (mon.0) 630 : audit [DBG] from='client.? 172.21.15.89:0/1781617151' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-15T07:07:53.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:07:53 smithi134 bash[67624]: audit 2024-07-15T07:07:53.155239+0000 mon.a (mon.0) 630 : audit [DBG] from='client.? 172.21.15.89:0/1781617151' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-15T07:07:54.372 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 07:07:54 smithi089 bash[72894]: ::ffff:172.21.15.134 - - [15/Jul/2024:07:07:54] "GET /metrics HTTP/1.1" 200 33974 "" "Prometheus/2.43.0" 2024-07-15T07:07:54.622 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:07:54 smithi089 bash[79810]: cluster 2024-07-15T07:07:52.960154+0000 mgr.y (mgr.25244) 417 : cluster [DBG] pgmap v362: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:07:54.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:07:54 smithi089 bash[82202]: cluster 2024-07-15T07:07:52.960154+0000 mgr.y (mgr.25244) 417 : cluster [DBG] pgmap v362: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:07:54.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:07:54 smithi134 bash[67624]: cluster 2024-07-15T07:07:52.960154+0000 mgr.y (mgr.25244) 417 : cluster [DBG] pgmap v362: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:07:56.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:07:56 smithi134 bash[67624]: cluster 2024-07-15T07:07:54.960902+0000 mgr.y (mgr.25244) 418 : cluster [DBG] pgmap v363: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:07:56.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:07:56 smithi089 bash[79810]: cluster 2024-07-15T07:07:54.960902+0000 mgr.y (mgr.25244) 418 : cluster [DBG] pgmap v363: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:07:56.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:07:56 smithi089 bash[82202]: cluster 2024-07-15T07:07:54.960902+0000 mgr.y (mgr.25244) 418 : cluster [DBG] pgmap v363: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:07:57.003 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph health --format=json 2024-07-15T07:07:58.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:07:58 smithi134 bash[67624]: cluster 2024-07-15T07:07:56.961835+0000 mgr.y (mgr.25244) 419 : cluster [DBG] pgmap v364: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:07:58.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:07:58 smithi089 bash[79810]: cluster 2024-07-15T07:07:56.961835+0000 mgr.y (mgr.25244) 419 : cluster [DBG] pgmap v364: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:07:58.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:07:58 smithi089 bash[82202]: cluster 2024-07-15T07:07:56.961835+0000 mgr.y (mgr.25244) 419 : cluster [DBG] pgmap v364: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:08:00.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:08:00 smithi134 bash[67624]: cluster 2024-07-15T07:07:58.962551+0000 mgr.y (mgr.25244) 420 : cluster [DBG] pgmap v365: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:08:00.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:08:00 smithi089 bash[79810]: cluster 2024-07-15T07:07:58.962551+0000 mgr.y (mgr.25244) 420 : cluster [DBG] pgmap v365: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:08:00.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:08:00 smithi089 bash[82202]: cluster 2024-07-15T07:07:58.962551+0000 mgr.y (mgr.25244) 420 : cluster [DBG] pgmap v365: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:08:01.790 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T07:08:02.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:08:02 smithi134 bash[67624]: cluster 2024-07-15T07:08:00.963559+0000 mgr.y (mgr.25244) 421 : cluster [DBG] pgmap v366: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:08:02.807 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:08:02 smithi134 bash[60729]: ts=2024-07-15T07:08:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T07:08:02.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:08:02 smithi089 bash[79810]: cluster 2024-07-15T07:08:00.963559+0000 mgr.y (mgr.25244) 421 : cluster [DBG] pgmap v366: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:08:02.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:08:02 smithi089 bash[82202]: cluster 2024-07-15T07:08:00.963559+0000 mgr.y (mgr.25244) 421 : cluster [DBG] pgmap v366: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:08:04.372 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 07:08:04 smithi089 bash[72894]: ::ffff:172.21.15.134 - - [15/Jul/2024:07:08:04] "GET /metrics HTTP/1.1" 200 33974 "" "Prometheus/2.43.0" 2024-07-15T07:08:04.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:08:04 smithi134 bash[67624]: cluster 2024-07-15T07:08:02.964475+0000 mgr.y (mgr.25244) 422 : cluster [DBG] pgmap v367: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:08:04.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:08:04 smithi089 bash[79810]: cluster 2024-07-15T07:08:02.964475+0000 mgr.y (mgr.25244) 422 : cluster [DBG] pgmap v367: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:08:04.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:08:04 smithi089 bash[82202]: cluster 2024-07-15T07:08:02.964475+0000 mgr.y (mgr.25244) 422 : cluster [DBG] pgmap v367: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:08:05.438 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-07-15T07:08:05.438 INFO:teuthology.orchestra.run.smithi089.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-07-15T07:08:05.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:08:05 smithi134 bash[67624]: audit 2024-07-15T07:08:04.800323+0000 mon.a (mon.0) 631 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:08:05.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:08:05 smithi089 bash[79810]: audit 2024-07-15T07:08:04.800323+0000 mon.a (mon.0) 631 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:08:05.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:08:05 smithi089 bash[82202]: audit 2024-07-15T07:08:04.800323+0000 mon.a (mon.0) 631 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:08:06.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:08:06 smithi134 bash[67624]: cluster 2024-07-15T07:08:04.965286+0000 mgr.y (mgr.25244) 423 : cluster [DBG] pgmap v368: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:08:06.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:08:06 smithi134 bash[67624]: audit 2024-07-15T07:08:05.440510+0000 mon.c (mon.2) 25 : audit [DBG] from='client.? 172.21.15.89:0/447867615' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-15T07:08:06.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:08:06 smithi089 bash[79810]: cluster 2024-07-15T07:08:04.965286+0000 mgr.y (mgr.25244) 423 : cluster [DBG] pgmap v368: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:08:06.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:08:06 smithi089 bash[79810]: audit 2024-07-15T07:08:05.440510+0000 mon.c (mon.2) 25 : audit [DBG] from='client.? 172.21.15.89:0/447867615' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-15T07:08:06.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:08:06 smithi089 bash[82202]: cluster 2024-07-15T07:08:04.965286+0000 mgr.y (mgr.25244) 423 : cluster [DBG] pgmap v368: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:08:06.873 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:08:06 smithi089 bash[82202]: audit 2024-07-15T07:08:05.440510+0000 mon.c (mon.2) 25 : audit [DBG] from='client.? 172.21.15.89:0/447867615' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-15T07:08:08.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:08:08 smithi134 bash[67624]: cluster 2024-07-15T07:08:06.966216+0000 mgr.y (mgr.25244) 424 : cluster [DBG] pgmap v369: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:08:08.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:08:08 smithi089 bash[79810]: cluster 2024-07-15T07:08:06.966216+0000 mgr.y (mgr.25244) 424 : cluster [DBG] pgmap v369: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:08:08.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:08:08 smithi089 bash[82202]: cluster 2024-07-15T07:08:06.966216+0000 mgr.y (mgr.25244) 424 : cluster [DBG] pgmap v369: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:08:09.162 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph health --format=json 2024-07-15T07:08:10.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:08:10 smithi134 bash[67624]: cluster 2024-07-15T07:08:08.967018+0000 mgr.y (mgr.25244) 425 : cluster [DBG] pgmap v370: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:08:10.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:08:10 smithi089 bash[79810]: cluster 2024-07-15T07:08:08.967018+0000 mgr.y (mgr.25244) 425 : cluster [DBG] pgmap v370: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:08:10.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:08:10 smithi089 bash[82202]: cluster 2024-07-15T07:08:08.967018+0000 mgr.y (mgr.25244) 425 : cluster [DBG] pgmap v370: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:08:12.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:08:12 smithi134 bash[67624]: cluster 2024-07-15T07:08:10.968129+0000 mgr.y (mgr.25244) 426 : cluster [DBG] pgmap v371: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:08:12.806 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:08:12 smithi134 bash[60729]: ts=2024-07-15T07:08:12.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T07:08:12.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:08:12 smithi089 bash[79810]: cluster 2024-07-15T07:08:10.968129+0000 mgr.y (mgr.25244) 426 : cluster [DBG] pgmap v371: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:08:12.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:08:12 smithi089 bash[82202]: cluster 2024-07-15T07:08:10.968129+0000 mgr.y (mgr.25244) 426 : cluster [DBG] pgmap v371: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:08:13.946 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T07:08:14.372 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 07:08:14 smithi089 bash[72894]: ::ffff:172.21.15.134 - - [15/Jul/2024:07:08:14] "GET /metrics HTTP/1.1" 200 33977 "" "Prometheus/2.43.0" 2024-07-15T07:08:14.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:08:14 smithi134 bash[67624]: cluster 2024-07-15T07:08:12.968864+0000 mgr.y (mgr.25244) 427 : cluster [DBG] pgmap v372: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:08:14.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:08:14 smithi089 bash[79810]: cluster 2024-07-15T07:08:12.968864+0000 mgr.y (mgr.25244) 427 : cluster [DBG] pgmap v372: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:08:14.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:08:14 smithi089 bash[82202]: cluster 2024-07-15T07:08:12.968864+0000 mgr.y (mgr.25244) 427 : cluster [DBG] pgmap v372: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:08:16.622 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:08:16 smithi089 bash[79810]: cluster 2024-07-15T07:08:14.969676+0000 mgr.y (mgr.25244) 428 : cluster [DBG] pgmap v373: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:08:16.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:08:16 smithi089 bash[82202]: cluster 2024-07-15T07:08:14.969676+0000 mgr.y (mgr.25244) 428 : cluster [DBG] pgmap v373: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:08:16.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:08:16 smithi134 bash[67624]: cluster 2024-07-15T07:08:14.969676+0000 mgr.y (mgr.25244) 428 : cluster [DBG] pgmap v373: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:08:17.707 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-07-15T07:08:17.707 INFO:teuthology.orchestra.run.smithi089.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-07-15T07:08:18.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:08:18 smithi134 bash[67624]: cluster 2024-07-15T07:08:16.970569+0000 mgr.y (mgr.25244) 429 : cluster [DBG] pgmap v374: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:08:18.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:08:18 smithi134 bash[67624]: audit 2024-07-15T07:08:17.710000+0000 mon.a (mon.0) 632 : audit [DBG] from='client.? 172.21.15.89:0/2290231131' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-15T07:08:18.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:08:18 smithi089 bash[79810]: cluster 2024-07-15T07:08:16.970569+0000 mgr.y (mgr.25244) 429 : cluster [DBG] pgmap v374: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:08:18.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:08:18 smithi089 bash[79810]: audit 2024-07-15T07:08:17.710000+0000 mon.a (mon.0) 632 : audit [DBG] from='client.? 172.21.15.89:0/2290231131' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-15T07:08:18.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:08:18 smithi089 bash[82202]: cluster 2024-07-15T07:08:16.970569+0000 mgr.y (mgr.25244) 429 : cluster [DBG] pgmap v374: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:08:18.873 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:08:18 smithi089 bash[82202]: audit 2024-07-15T07:08:17.710000+0000 mon.a (mon.0) 632 : audit [DBG] from='client.? 172.21.15.89:0/2290231131' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-15T07:08:20.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:08:20 smithi134 bash[67624]: cluster 2024-07-15T07:08:18.971403+0000 mgr.y (mgr.25244) 430 : cluster [DBG] pgmap v375: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:08:20.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:08:20 smithi134 bash[67624]: audit 2024-07-15T07:08:19.800801+0000 mon.a (mon.0) 633 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:08:20.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:08:20 smithi089 bash[79810]: cluster 2024-07-15T07:08:18.971403+0000 mgr.y (mgr.25244) 430 : cluster [DBG] pgmap v375: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:08:20.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:08:20 smithi089 bash[79810]: audit 2024-07-15T07:08:19.800801+0000 mon.a (mon.0) 633 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:08:20.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:08:20 smithi089 bash[82202]: cluster 2024-07-15T07:08:18.971403+0000 mgr.y (mgr.25244) 430 : cluster [DBG] pgmap v375: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:08:20.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:08:20 smithi089 bash[82202]: audit 2024-07-15T07:08:19.800801+0000 mon.a (mon.0) 633 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:08:21.455 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph health --format=json 2024-07-15T07:08:22.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:08:22 smithi134 bash[67624]: cluster 2024-07-15T07:08:20.972510+0000 mgr.y (mgr.25244) 431 : cluster [DBG] pgmap v376: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:08:22.806 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:08:22 smithi134 bash[60729]: ts=2024-07-15T07:08:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T07:08:22.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:08:22 smithi089 bash[79810]: cluster 2024-07-15T07:08:20.972510+0000 mgr.y (mgr.25244) 431 : cluster [DBG] pgmap v376: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:08:22.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:08:22 smithi089 bash[82202]: cluster 2024-07-15T07:08:20.972510+0000 mgr.y (mgr.25244) 431 : cluster [DBG] pgmap v376: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:08:24.464 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 07:08:24 smithi089 bash[72894]: ::ffff:172.21.15.134 - - [15/Jul/2024:07:08:24] "GET /metrics HTTP/1.1" 200 33977 "" "Prometheus/2.43.0" 2024-07-15T07:08:24.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:08:24 smithi134 bash[67624]: cluster 2024-07-15T07:08:22.973333+0000 mgr.y (mgr.25244) 432 : cluster [DBG] pgmap v377: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:08:24.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:08:24 smithi089 bash[79810]: cluster 2024-07-15T07:08:22.973333+0000 mgr.y (mgr.25244) 432 : cluster [DBG] pgmap v377: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:08:24.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:08:24 smithi089 bash[82202]: cluster 2024-07-15T07:08:22.973333+0000 mgr.y (mgr.25244) 432 : cluster [DBG] pgmap v377: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:08:26.236 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T07:08:26.622 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:08:26 smithi089 bash[79810]: cluster 2024-07-15T07:08:24.974246+0000 mgr.y (mgr.25244) 433 : cluster [DBG] pgmap v378: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:08:26.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:08:26 smithi089 bash[82202]: cluster 2024-07-15T07:08:24.974246+0000 mgr.y (mgr.25244) 433 : cluster [DBG] pgmap v378: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:08:26.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:08:26 smithi134 bash[67624]: cluster 2024-07-15T07:08:24.974246+0000 mgr.y (mgr.25244) 433 : cluster [DBG] pgmap v378: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:08:28.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:08:28 smithi134 bash[67624]: cluster 2024-07-15T07:08:26.975371+0000 mgr.y (mgr.25244) 434 : cluster [DBG] pgmap v379: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:08:28.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:08:28 smithi089 bash[79810]: cluster 2024-07-15T07:08:26.975371+0000 mgr.y (mgr.25244) 434 : cluster [DBG] pgmap v379: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:08:28.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:08:28 smithi089 bash[82202]: cluster 2024-07-15T07:08:26.975371+0000 mgr.y (mgr.25244) 434 : cluster [DBG] pgmap v379: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:08:30.181 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-07-15T07:08:30.182 INFO:teuthology.orchestra.run.smithi089.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-07-15T07:08:30.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:08:30 smithi134 bash[67624]: cluster 2024-07-15T07:08:28.976157+0000 mgr.y (mgr.25244) 435 : cluster [DBG] pgmap v380: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:08:30.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:08:30 smithi134 bash[67624]: audit 2024-07-15T07:08:30.184420+0000 mon.c (mon.2) 26 : audit [DBG] from='client.? 172.21.15.89:0/2547528623' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-15T07:08:30.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:08:30 smithi089 bash[82202]: cluster 2024-07-15T07:08:28.976157+0000 mgr.y (mgr.25244) 435 : cluster [DBG] pgmap v380: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:08:30.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:08:30 smithi089 bash[82202]: audit 2024-07-15T07:08:30.184420+0000 mon.c (mon.2) 26 : audit [DBG] from='client.? 172.21.15.89:0/2547528623' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-15T07:08:30.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:08:30 smithi089 bash[79810]: cluster 2024-07-15T07:08:28.976157+0000 mgr.y (mgr.25244) 435 : cluster [DBG] pgmap v380: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:08:30.873 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:08:30 smithi089 bash[79810]: audit 2024-07-15T07:08:30.184420+0000 mon.c (mon.2) 26 : audit [DBG] from='client.? 172.21.15.89:0/2547528623' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-15T07:08:32.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:08:32 smithi134 bash[67624]: cluster 2024-07-15T07:08:30.977127+0000 mgr.y (mgr.25244) 436 : cluster [DBG] pgmap v381: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:08:32.807 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:08:32 smithi134 bash[60729]: ts=2024-07-15T07:08:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T07:08:32.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:08:32 smithi089 bash[82202]: cluster 2024-07-15T07:08:30.977127+0000 mgr.y (mgr.25244) 436 : cluster [DBG] pgmap v381: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:08:32.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:08:32 smithi089 bash[79810]: cluster 2024-07-15T07:08:30.977127+0000 mgr.y (mgr.25244) 436 : cluster [DBG] pgmap v381: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:08:33.936 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph health --format=json 2024-07-15T07:08:34.372 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 07:08:34 smithi089 bash[72894]: ::ffff:172.21.15.134 - - [15/Jul/2024:07:08:34] "GET /metrics HTTP/1.1" 200 33977 "" "Prometheus/2.43.0" 2024-07-15T07:08:34.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:08:34 smithi134 bash[67624]: cluster 2024-07-15T07:08:32.977870+0000 mgr.y (mgr.25244) 437 : cluster [DBG] pgmap v382: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:08:34.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:08:34 smithi089 bash[79810]: cluster 2024-07-15T07:08:32.977870+0000 mgr.y (mgr.25244) 437 : cluster [DBG] pgmap v382: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:08:34.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:08:34 smithi089 bash[82202]: cluster 2024-07-15T07:08:32.977870+0000 mgr.y (mgr.25244) 437 : cluster [DBG] pgmap v382: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:08:35.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:08:35 smithi134 bash[67624]: audit 2024-07-15T07:08:34.801284+0000 mon.a (mon.0) 634 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:08:35.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:08:35 smithi089 bash[79810]: audit 2024-07-15T07:08:34.801284+0000 mon.a (mon.0) 634 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:08:35.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:08:35 smithi089 bash[82202]: audit 2024-07-15T07:08:34.801284+0000 mon.a (mon.0) 634 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:08:36.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:08:36 smithi134 bash[67624]: cluster 2024-07-15T07:08:34.978672+0000 mgr.y (mgr.25244) 438 : cluster [DBG] pgmap v383: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:08:36.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:08:36 smithi089 bash[79810]: cluster 2024-07-15T07:08:34.978672+0000 mgr.y (mgr.25244) 438 : cluster [DBG] pgmap v383: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:08:36.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:08:36 smithi089 bash[82202]: cluster 2024-07-15T07:08:34.978672+0000 mgr.y (mgr.25244) 438 : cluster [DBG] pgmap v383: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:08:38.721 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T07:08:38.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:08:38 smithi134 bash[67624]: cluster 2024-07-15T07:08:36.979680+0000 mgr.y (mgr.25244) 439 : cluster [DBG] pgmap v384: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:08:38.821 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:08:38 smithi089 bash[79810]: cluster 2024-07-15T07:08:36.979680+0000 mgr.y (mgr.25244) 439 : cluster [DBG] pgmap v384: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:08:38.821 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:08:38 smithi089 bash[82202]: cluster 2024-07-15T07:08:36.979680+0000 mgr.y (mgr.25244) 439 : cluster [DBG] pgmap v384: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:08:40.622 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:08:40 smithi089 bash[79810]: cluster 2024-07-15T07:08:38.980446+0000 mgr.y (mgr.25244) 440 : cluster [DBG] pgmap v385: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:08:40.625 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:08:40 smithi089 bash[82202]: cluster 2024-07-15T07:08:38.980446+0000 mgr.y (mgr.25244) 440 : cluster [DBG] pgmap v385: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:08:40.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:08:40 smithi134 bash[67624]: cluster 2024-07-15T07:08:38.980446+0000 mgr.y (mgr.25244) 440 : cluster [DBG] pgmap v385: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:08:42.441 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-07-15T07:08:42.442 INFO:teuthology.orchestra.run.smithi089.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-07-15T07:08:42.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:08:42 smithi134 bash[67624]: cluster 2024-07-15T07:08:40.981444+0000 mgr.y (mgr.25244) 441 : cluster [DBG] pgmap v386: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:08:42.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:08:42 smithi134 bash[67624]: audit 2024-07-15T07:08:42.444495+0000 mon.a (mon.0) 635 : audit [DBG] from='client.? 172.21.15.89:0/3843237876' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-15T07:08:42.807 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:08:42 smithi134 bash[60729]: ts=2024-07-15T07:08:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T07:08:42.875 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:08:42 smithi089 bash[79810]: cluster 2024-07-15T07:08:40.981444+0000 mgr.y (mgr.25244) 441 : cluster [DBG] pgmap v386: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:08:42.875 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:08:42 smithi089 bash[79810]: audit 2024-07-15T07:08:42.444495+0000 mon.a (mon.0) 635 : audit [DBG] from='client.? 172.21.15.89:0/3843237876' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-15T07:08:42.875 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:08:42 smithi089 bash[82202]: cluster 2024-07-15T07:08:40.981444+0000 mgr.y (mgr.25244) 441 : cluster [DBG] pgmap v386: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:08:42.875 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:08:42 smithi089 bash[82202]: audit 2024-07-15T07:08:42.444495+0000 mon.a (mon.0) 635 : audit [DBG] from='client.? 172.21.15.89:0/3843237876' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-15T07:08:44.511 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 07:08:44 smithi089 bash[72894]: ::ffff:172.21.15.134 - - [15/Jul/2024:07:08:44] "GET /metrics HTTP/1.1" 200 33978 "" "Prometheus/2.43.0" 2024-07-15T07:08:44.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:08:44 smithi134 bash[67624]: cluster 2024-07-15T07:08:42.982184+0000 mgr.y (mgr.25244) 442 : cluster [DBG] pgmap v387: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:08:44.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:08:44 smithi134 bash[67624]: audit 2024-07-15T07:08:43.754999+0000 mon.a (mon.0) 636 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T07:08:44.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:08:44 smithi134 bash[67624]: audit 2024-07-15T07:08:44.121274+0000 mon.a (mon.0) 637 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:08:44.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:08:44 smithi134 bash[67624]: audit 2024-07-15T07:08:44.126516+0000 mon.a (mon.0) 638 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:08:44.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:08:44 smithi089 bash[79810]: cluster 2024-07-15T07:08:42.982184+0000 mgr.y (mgr.25244) 442 : cluster [DBG] pgmap v387: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:08:44.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:08:44 smithi089 bash[79810]: audit 2024-07-15T07:08:43.754999+0000 mon.a (mon.0) 636 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T07:08:44.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:08:44 smithi089 bash[79810]: audit 2024-07-15T07:08:44.121274+0000 mon.a (mon.0) 637 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:08:44.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:08:44 smithi089 bash[79810]: audit 2024-07-15T07:08:44.126516+0000 mon.a (mon.0) 638 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:08:44.873 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:08:44 smithi089 bash[82202]: cluster 2024-07-15T07:08:42.982184+0000 mgr.y (mgr.25244) 442 : cluster [DBG] pgmap v387: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:08:44.873 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:08:44 smithi089 bash[82202]: audit 2024-07-15T07:08:43.754999+0000 mon.a (mon.0) 636 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T07:08:44.873 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:08:44 smithi089 bash[82202]: audit 2024-07-15T07:08:44.121274+0000 mon.a (mon.0) 637 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:08:44.873 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:08:44 smithi089 bash[82202]: audit 2024-07-15T07:08:44.126516+0000 mon.a (mon.0) 638 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:08:46.118 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph health --format=json 2024-07-15T07:08:46.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:08:46 smithi134 bash[67624]: cluster 2024-07-15T07:08:44.982991+0000 mgr.y (mgr.25244) 443 : cluster [DBG] pgmap v388: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:08:46.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:08:46 smithi089 bash[79810]: cluster 2024-07-15T07:08:44.982991+0000 mgr.y (mgr.25244) 443 : cluster [DBG] pgmap v388: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:08:46.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:08:46 smithi089 bash[82202]: cluster 2024-07-15T07:08:44.982991+0000 mgr.y (mgr.25244) 443 : cluster [DBG] pgmap v388: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:08:48.806 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:08:48 smithi134 bash[67624]: cluster 2024-07-15T07:08:46.983581+0000 mgr.y (mgr.25244) 444 : cluster [DBG] pgmap v389: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:08:48.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:08:48 smithi089 bash[79810]: cluster 2024-07-15T07:08:46.983581+0000 mgr.y (mgr.25244) 444 : cluster [DBG] pgmap v389: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:08:48.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:08:48 smithi089 bash[82202]: cluster 2024-07-15T07:08:46.983581+0000 mgr.y (mgr.25244) 444 : cluster [DBG] pgmap v389: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:08:49.982 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T07:08:50.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:08:50 smithi089 bash[79810]: cluster 2024-07-15T07:08:48.984129+0000 mgr.y (mgr.25244) 445 : cluster [DBG] pgmap v390: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:08:50.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:08:50 smithi089 bash[79810]: audit 2024-07-15T07:08:49.605000+0000 mon.a (mon.0) 639 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:08:50.873 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:08:50 smithi089 bash[79810]: audit 2024-07-15T07:08:49.614794+0000 mon.a (mon.0) 640 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:08:50.873 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:08:50 smithi089 bash[79810]: audit 2024-07-15T07:08:49.678689+0000 mon.a (mon.0) 641 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:08:50.873 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:08:50 smithi089 bash[79810]: audit 2024-07-15T07:08:49.684890+0000 mon.a (mon.0) 642 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:08:50.873 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:08:50 smithi089 bash[79810]: audit 2024-07-15T07:08:49.801502+0000 mon.a (mon.0) 643 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:08:50.873 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:08:50 smithi089 bash[79810]: audit 2024-07-15T07:08:50.149136+0000 mon.a (mon.0) 644 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T07:08:50.873 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:08:50 smithi089 bash[79810]: audit 2024-07-15T07:08:50.150300+0000 mon.a (mon.0) 645 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T07:08:50.873 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:08:50 smithi089 bash[79810]: audit 2024-07-15T07:08:50.158088+0000 mon.a (mon.0) 646 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:08:50.873 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:08:50 smithi089 bash[82202]: cluster 2024-07-15T07:08:48.984129+0000 mgr.y (mgr.25244) 445 : cluster [DBG] pgmap v390: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:08:50.874 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:08:50 smithi089 bash[82202]: audit 2024-07-15T07:08:49.605000+0000 mon.a (mon.0) 639 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:08:50.874 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:08:50 smithi089 bash[82202]: audit 2024-07-15T07:08:49.614794+0000 mon.a (mon.0) 640 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:08:50.874 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:08:50 smithi089 bash[82202]: audit 2024-07-15T07:08:49.678689+0000 mon.a (mon.0) 641 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:08:50.874 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:08:50 smithi089 bash[82202]: audit 2024-07-15T07:08:49.684890+0000 mon.a (mon.0) 642 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:08:50.874 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:08:50 smithi089 bash[82202]: audit 2024-07-15T07:08:49.801502+0000 mon.a (mon.0) 643 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:08:50.874 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:08:50 smithi089 bash[82202]: audit 2024-07-15T07:08:50.149136+0000 mon.a (mon.0) 644 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T07:08:50.874 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:08:50 smithi089 bash[82202]: audit 2024-07-15T07:08:50.150300+0000 mon.a (mon.0) 645 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T07:08:50.874 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:08:50 smithi089 bash[82202]: audit 2024-07-15T07:08:50.158088+0000 mon.a (mon.0) 646 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:08:51.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:08:50 smithi134 bash[67624]: cluster 2024-07-15T07:08:48.984129+0000 mgr.y (mgr.25244) 445 : cluster [DBG] pgmap v390: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:08:51.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:08:50 smithi134 bash[67624]: audit 2024-07-15T07:08:49.605000+0000 mon.a (mon.0) 639 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:08:51.057 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:08:50 smithi134 bash[67624]: audit 2024-07-15T07:08:49.614794+0000 mon.a (mon.0) 640 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:08:51.057 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:08:50 smithi134 bash[67624]: audit 2024-07-15T07:08:49.678689+0000 mon.a (mon.0) 641 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:08:51.057 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:08:50 smithi134 bash[67624]: audit 2024-07-15T07:08:49.684890+0000 mon.a (mon.0) 642 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:08:51.057 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:08:50 smithi134 bash[67624]: audit 2024-07-15T07:08:49.801502+0000 mon.a (mon.0) 643 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:08:51.057 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:08:50 smithi134 bash[67624]: audit 2024-07-15T07:08:50.149136+0000 mon.a (mon.0) 644 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T07:08:51.057 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:08:50 smithi134 bash[67624]: audit 2024-07-15T07:08:50.150300+0000 mon.a (mon.0) 645 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T07:08:51.057 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:08:50 smithi134 bash[67624]: audit 2024-07-15T07:08:50.158088+0000 mon.a (mon.0) 646 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:08:52.606 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:08:52 smithi134 bash[60729]: ts=2024-07-15T07:08:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T07:08:52.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:08:52 smithi089 bash[82202]: cluster 2024-07-15T07:08:50.985079+0000 mgr.y (mgr.25244) 446 : cluster [DBG] pgmap v391: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:08:52.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:08:52 smithi089 bash[79810]: cluster 2024-07-15T07:08:50.985079+0000 mgr.y (mgr.25244) 446 : cluster [DBG] pgmap v391: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:08:53.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:08:52 smithi134 bash[67624]: cluster 2024-07-15T07:08:50.985079+0000 mgr.y (mgr.25244) 446 : cluster [DBG] pgmap v391: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:08:53.805 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-07-15T07:08:53.805 INFO:teuthology.orchestra.run.smithi089.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-07-15T07:08:54.544 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 07:08:54 smithi089 bash[72894]: ::ffff:172.21.15.134 - - [15/Jul/2024:07:08:54] "GET /metrics HTTP/1.1" 200 33977 "" "Prometheus/2.43.0" 2024-07-15T07:08:54.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:08:54 smithi089 bash[79810]: cluster 2024-07-15T07:08:52.985818+0000 mgr.y (mgr.25244) 447 : cluster [DBG] pgmap v392: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:08:54.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:08:54 smithi089 bash[79810]: audit 2024-07-15T07:08:53.807624+0000 mon.c (mon.2) 27 : audit [DBG] from='client.? 172.21.15.89:0/1788018728' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-15T07:08:54.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:08:54 smithi089 bash[82202]: cluster 2024-07-15T07:08:52.985818+0000 mgr.y (mgr.25244) 447 : cluster [DBG] pgmap v392: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:08:54.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:08:54 smithi089 bash[82202]: audit 2024-07-15T07:08:53.807624+0000 mon.c (mon.2) 27 : audit [DBG] from='client.? 172.21.15.89:0/1788018728' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-15T07:08:55.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:08:54 smithi134 bash[67624]: cluster 2024-07-15T07:08:52.985818+0000 mgr.y (mgr.25244) 447 : cluster [DBG] pgmap v392: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:08:55.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:08:54 smithi134 bash[67624]: audit 2024-07-15T07:08:53.807624+0000 mon.c (mon.2) 27 : audit [DBG] from='client.? 172.21.15.89:0/1788018728' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-15T07:08:56.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:08:56 smithi089 bash[79810]: cluster 2024-07-15T07:08:54.986604+0000 mgr.y (mgr.25244) 448 : cluster [DBG] pgmap v393: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:08:56.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:08:56 smithi089 bash[82202]: cluster 2024-07-15T07:08:54.986604+0000 mgr.y (mgr.25244) 448 : cluster [DBG] pgmap v393: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:08:57.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:08:56 smithi134 bash[67624]: cluster 2024-07-15T07:08:54.986604+0000 mgr.y (mgr.25244) 448 : cluster [DBG] pgmap v393: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:08:57.547 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph health --format=json 2024-07-15T07:08:58.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:08:58 smithi089 bash[79810]: cluster 2024-07-15T07:08:56.987579+0000 mgr.y (mgr.25244) 449 : cluster [DBG] pgmap v394: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:08:58.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:08:58 smithi089 bash[82202]: cluster 2024-07-15T07:08:56.987579+0000 mgr.y (mgr.25244) 449 : cluster [DBG] pgmap v394: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:08:59.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:08:58 smithi134 bash[67624]: cluster 2024-07-15T07:08:56.987579+0000 mgr.y (mgr.25244) 449 : cluster [DBG] pgmap v394: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:09:00.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:09:00 smithi089 bash[79810]: cluster 2024-07-15T07:08:58.988332+0000 mgr.y (mgr.25244) 450 : cluster [DBG] pgmap v395: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:09:00.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:09:00 smithi089 bash[82202]: cluster 2024-07-15T07:08:58.988332+0000 mgr.y (mgr.25244) 450 : cluster [DBG] pgmap v395: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:09:01.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:09:00 smithi134 bash[67624]: cluster 2024-07-15T07:08:58.988332+0000 mgr.y (mgr.25244) 450 : cluster [DBG] pgmap v395: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:09:02.336 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T07:09:02.625 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:09:02 smithi134 bash[60729]: ts=2024-07-15T07:09:02.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T07:09:03.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:09:02 smithi134 bash[67624]: cluster 2024-07-15T07:09:00.989295+0000 mgr.y (mgr.25244) 451 : cluster [DBG] pgmap v396: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:09:03.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:09:02 smithi089 bash[79810]: cluster 2024-07-15T07:09:00.989295+0000 mgr.y (mgr.25244) 451 : cluster [DBG] pgmap v396: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:09:03.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:09:02 smithi089 bash[82202]: cluster 2024-07-15T07:09:00.989295+0000 mgr.y (mgr.25244) 451 : cluster [DBG] pgmap v396: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:09:04.372 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 07:09:04 smithi089 bash[72894]: ::ffff:172.21.15.134 - - [15/Jul/2024:07:09:04] "GET /metrics HTTP/1.1" 200 33977 "" "Prometheus/2.43.0" 2024-07-15T07:09:04.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:09:04 smithi089 bash[79810]: cluster 2024-07-15T07:09:02.989989+0000 mgr.y (mgr.25244) 452 : cluster [DBG] pgmap v397: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:09:04.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:09:04 smithi089 bash[82202]: cluster 2024-07-15T07:09:02.989989+0000 mgr.y (mgr.25244) 452 : cluster [DBG] pgmap v397: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:09:05.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:09:04 smithi134 bash[67624]: cluster 2024-07-15T07:09:02.989989+0000 mgr.y (mgr.25244) 452 : cluster [DBG] pgmap v397: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:09:06.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:09:05 smithi134 bash[67624]: audit 2024-07-15T07:09:04.802072+0000 mon.a (mon.0) 647 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:09:06.122 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-07-15T07:09:06.122 INFO:teuthology.orchestra.run.smithi089.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-07-15T07:09:06.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:09:05 smithi089 bash[79810]: audit 2024-07-15T07:09:04.802072+0000 mon.a (mon.0) 647 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:09:06.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:09:05 smithi089 bash[82202]: audit 2024-07-15T07:09:04.802072+0000 mon.a (mon.0) 647 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:09:06.933 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:09:06 smithi089 bash[79810]: cluster 2024-07-15T07:09:04.990871+0000 mgr.y (mgr.25244) 453 : cluster [DBG] pgmap v398: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:09:06.933 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:09:06 smithi089 bash[79810]: audit 2024-07-15T07:09:06.124592+0000 mon.c (mon.2) 28 : audit [DBG] from='client.? 172.21.15.89:0/504553548' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-15T07:09:06.933 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:09:06 smithi089 bash[82202]: cluster 2024-07-15T07:09:04.990871+0000 mgr.y (mgr.25244) 453 : cluster [DBG] pgmap v398: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:09:06.933 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:09:06 smithi089 bash[82202]: audit 2024-07-15T07:09:06.124592+0000 mon.c (mon.2) 28 : audit [DBG] from='client.? 172.21.15.89:0/504553548' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-15T07:09:07.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:09:06 smithi134 bash[67624]: cluster 2024-07-15T07:09:04.990871+0000 mgr.y (mgr.25244) 453 : cluster [DBG] pgmap v398: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:09:07.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:09:06 smithi134 bash[67624]: audit 2024-07-15T07:09:06.124592+0000 mon.c (mon.2) 28 : audit [DBG] from='client.? 172.21.15.89:0/504553548' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-15T07:09:09.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:09:08 smithi134 bash[67624]: cluster 2024-07-15T07:09:06.991717+0000 mgr.y (mgr.25244) 454 : cluster [DBG] pgmap v399: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:09:09.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:09:08 smithi089 bash[79810]: cluster 2024-07-15T07:09:06.991717+0000 mgr.y (mgr.25244) 454 : cluster [DBG] pgmap v399: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:09:09.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:09:08 smithi089 bash[82202]: cluster 2024-07-15T07:09:06.991717+0000 mgr.y (mgr.25244) 454 : cluster [DBG] pgmap v399: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:09:09.935 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph health --format=json 2024-07-15T07:09:11.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:09:10 smithi134 bash[67624]: cluster 2024-07-15T07:09:08.992414+0000 mgr.y (mgr.25244) 455 : cluster [DBG] pgmap v400: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:09:11.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:09:10 smithi089 bash[79810]: cluster 2024-07-15T07:09:08.992414+0000 mgr.y (mgr.25244) 455 : cluster [DBG] pgmap v400: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:09:11.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:09:10 smithi089 bash[82202]: cluster 2024-07-15T07:09:08.992414+0000 mgr.y (mgr.25244) 455 : cluster [DBG] pgmap v400: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:09:12.656 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:09:12 smithi134 bash[60729]: ts=2024-07-15T07:09:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T07:09:13.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:09:12 smithi134 bash[67624]: cluster 2024-07-15T07:09:10.993353+0000 mgr.y (mgr.25244) 456 : cluster [DBG] pgmap v401: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:09:13.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:09:12 smithi089 bash[79810]: cluster 2024-07-15T07:09:10.993353+0000 mgr.y (mgr.25244) 456 : cluster [DBG] pgmap v401: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:09:13.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:09:12 smithi089 bash[82202]: cluster 2024-07-15T07:09:10.993353+0000 mgr.y (mgr.25244) 456 : cluster [DBG] pgmap v401: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:09:14.622 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 07:09:14 smithi089 bash[72894]: ::ffff:172.21.15.134 - - [15/Jul/2024:07:09:14] "GET /metrics HTTP/1.1" 200 33978 "" "Prometheus/2.43.0" 2024-07-15T07:09:14.721 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T07:09:15.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:09:14 smithi134 bash[67624]: cluster 2024-07-15T07:09:12.994054+0000 mgr.y (mgr.25244) 457 : cluster [DBG] pgmap v402: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:09:15.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:09:14 smithi089 bash[79810]: cluster 2024-07-15T07:09:12.994054+0000 mgr.y (mgr.25244) 457 : cluster [DBG] pgmap v402: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:09:15.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:09:14 smithi089 bash[82202]: cluster 2024-07-15T07:09:12.994054+0000 mgr.y (mgr.25244) 457 : cluster [DBG] pgmap v402: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:09:16.937 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:09:16 smithi089 bash[82202]: cluster 2024-07-15T07:09:14.994843+0000 mgr.y (mgr.25244) 458 : cluster [DBG] pgmap v403: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:09:16.937 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:09:16 smithi089 bash[79810]: cluster 2024-07-15T07:09:14.994843+0000 mgr.y (mgr.25244) 458 : cluster [DBG] pgmap v403: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:09:17.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:09:16 smithi134 bash[67624]: cluster 2024-07-15T07:09:14.994843+0000 mgr.y (mgr.25244) 458 : cluster [DBG] pgmap v403: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:09:18.434 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-07-15T07:09:18.434 INFO:teuthology.orchestra.run.smithi089.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-07-15T07:09:18.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:09:18 smithi089 bash[79810]: cluster 2024-07-15T07:09:16.995837+0000 mgr.y (mgr.25244) 459 : cluster [DBG] pgmap v404: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:09:18.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:09:18 smithi089 bash[79810]: audit 2024-07-15T07:09:18.436630+0000 mon.c (mon.2) 29 : audit [DBG] from='client.? 172.21.15.89:0/190887758' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-15T07:09:18.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:09:18 smithi089 bash[82202]: cluster 2024-07-15T07:09:16.995837+0000 mgr.y (mgr.25244) 459 : cluster [DBG] pgmap v404: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:09:18.873 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:09:18 smithi089 bash[82202]: audit 2024-07-15T07:09:18.436630+0000 mon.c (mon.2) 29 : audit [DBG] from='client.? 172.21.15.89:0/190887758' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-15T07:09:19.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:09:18 smithi134 bash[67624]: cluster 2024-07-15T07:09:16.995837+0000 mgr.y (mgr.25244) 459 : cluster [DBG] pgmap v404: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:09:19.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:09:18 smithi134 bash[67624]: audit 2024-07-15T07:09:18.436630+0000 mon.c (mon.2) 29 : audit [DBG] from='client.? 172.21.15.89:0/190887758' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-15T07:09:21.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:09:20 smithi134 bash[67624]: cluster 2024-07-15T07:09:18.996558+0000 mgr.y (mgr.25244) 460 : cluster [DBG] pgmap v405: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:09:21.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:09:20 smithi134 bash[67624]: audit 2024-07-15T07:09:19.802515+0000 mon.a (mon.0) 648 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:09:21.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:09:20 smithi089 bash[79810]: cluster 2024-07-15T07:09:18.996558+0000 mgr.y (mgr.25244) 460 : cluster [DBG] pgmap v405: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:09:21.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:09:20 smithi089 bash[79810]: audit 2024-07-15T07:09:19.802515+0000 mon.a (mon.0) 648 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:09:21.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:09:20 smithi089 bash[82202]: cluster 2024-07-15T07:09:18.996558+0000 mgr.y (mgr.25244) 460 : cluster [DBG] pgmap v405: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:09:21.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:09:20 smithi089 bash[82202]: audit 2024-07-15T07:09:19.802515+0000 mon.a (mon.0) 648 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:09:22.205 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph health --format=json 2024-07-15T07:09:22.677 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:09:22 smithi134 bash[60729]: ts=2024-07-15T07:09:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T07:09:23.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:09:22 smithi134 bash[67624]: cluster 2024-07-15T07:09:20.997539+0000 mgr.y (mgr.25244) 461 : cluster [DBG] pgmap v406: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:09:23.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:09:22 smithi089 bash[79810]: cluster 2024-07-15T07:09:20.997539+0000 mgr.y (mgr.25244) 461 : cluster [DBG] pgmap v406: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:09:23.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:09:22 smithi089 bash[82202]: cluster 2024-07-15T07:09:20.997539+0000 mgr.y (mgr.25244) 461 : cluster [DBG] pgmap v406: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:09:24.622 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 07:09:24 smithi089 bash[72894]: ::ffff:172.21.15.134 - - [15/Jul/2024:07:09:24] "GET /metrics HTTP/1.1" 200 33975 "" "Prometheus/2.43.0" 2024-07-15T07:09:25.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:09:24 smithi134 bash[67624]: cluster 2024-07-15T07:09:22.998279+0000 mgr.y (mgr.25244) 462 : cluster [DBG] pgmap v407: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:09:25.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:09:24 smithi089 bash[79810]: cluster 2024-07-15T07:09:22.998279+0000 mgr.y (mgr.25244) 462 : cluster [DBG] pgmap v407: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:09:25.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:09:24 smithi089 bash[82202]: cluster 2024-07-15T07:09:22.998279+0000 mgr.y (mgr.25244) 462 : cluster [DBG] pgmap v407: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:09:26.982 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T07:09:27.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:09:26 smithi134 bash[67624]: cluster 2024-07-15T07:09:24.998874+0000 mgr.y (mgr.25244) 463 : cluster [DBG] pgmap v408: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:09:27.080 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:09:26 smithi089 bash[79810]: cluster 2024-07-15T07:09:24.998874+0000 mgr.y (mgr.25244) 463 : cluster [DBG] pgmap v408: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:09:27.080 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:09:26 smithi089 bash[82202]: cluster 2024-07-15T07:09:24.998874+0000 mgr.y (mgr.25244) 463 : cluster [DBG] pgmap v408: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:09:28.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:09:28 smithi089 bash[79810]: cluster 2024-07-15T07:09:26.999570+0000 mgr.y (mgr.25244) 464 : cluster [DBG] pgmap v409: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:09:28.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:09:28 smithi089 bash[82202]: cluster 2024-07-15T07:09:26.999570+0000 mgr.y (mgr.25244) 464 : cluster [DBG] pgmap v409: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:09:29.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:09:28 smithi134 bash[67624]: cluster 2024-07-15T07:09:26.999570+0000 mgr.y (mgr.25244) 464 : cluster [DBG] pgmap v409: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:09:30.700 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-07-15T07:09:30.700 INFO:teuthology.orchestra.run.smithi089.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-07-15T07:09:31.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:09:30 smithi134 bash[67624]: cluster 2024-07-15T07:09:29.000359+0000 mgr.y (mgr.25244) 465 : cluster [DBG] pgmap v410: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:09:31.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:09:30 smithi089 bash[79810]: cluster 2024-07-15T07:09:29.000359+0000 mgr.y (mgr.25244) 465 : cluster [DBG] pgmap v410: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:09:31.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:09:30 smithi089 bash[82202]: cluster 2024-07-15T07:09:29.000359+0000 mgr.y (mgr.25244) 465 : cluster [DBG] pgmap v410: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:09:31.873 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:09:31 smithi089 bash[79810]: audit 2024-07-15T07:09:30.702866+0000 mon.a (mon.0) 649 : audit [DBG] from='client.? 172.21.15.89:0/2719993203' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-15T07:09:31.873 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:09:31 smithi089 bash[82202]: audit 2024-07-15T07:09:30.702866+0000 mon.a (mon.0) 649 : audit [DBG] from='client.? 172.21.15.89:0/2719993203' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-15T07:09:32.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:09:31 smithi134 bash[67624]: audit 2024-07-15T07:09:30.702866+0000 mon.a (mon.0) 649 : audit [DBG] from='client.? 172.21.15.89:0/2719993203' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-15T07:09:32.709 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:09:32 smithi134 bash[60729]: ts=2024-07-15T07:09:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T07:09:33.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:09:32 smithi134 bash[67624]: cluster 2024-07-15T07:09:31.001355+0000 mgr.y (mgr.25244) 466 : cluster [DBG] pgmap v411: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:09:33.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:09:32 smithi089 bash[79810]: cluster 2024-07-15T07:09:31.001355+0000 mgr.y (mgr.25244) 466 : cluster [DBG] pgmap v411: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:09:33.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:09:32 smithi089 bash[82202]: cluster 2024-07-15T07:09:31.001355+0000 mgr.y (mgr.25244) 466 : cluster [DBG] pgmap v411: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:09:34.455 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph health --format=json 2024-07-15T07:09:34.467 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 07:09:34 smithi089 bash[72894]: ::ffff:172.21.15.134 - - [15/Jul/2024:07:09:34] "GET /metrics HTTP/1.1" 200 33975 "" "Prometheus/2.43.0" 2024-07-15T07:09:34.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:09:34 smithi089 bash[79810]: cluster 2024-07-15T07:09:33.002155+0000 mgr.y (mgr.25244) 467 : cluster [DBG] pgmap v412: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:09:34.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:09:34 smithi089 bash[82202]: cluster 2024-07-15T07:09:33.002155+0000 mgr.y (mgr.25244) 467 : cluster [DBG] pgmap v412: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:09:35.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:09:34 smithi134 bash[67624]: cluster 2024-07-15T07:09:33.002155+0000 mgr.y (mgr.25244) 467 : cluster [DBG] pgmap v412: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:09:36.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:09:35 smithi134 bash[67624]: audit 2024-07-15T07:09:34.802904+0000 mon.a (mon.0) 650 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:09:36.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:09:35 smithi089 bash[79810]: audit 2024-07-15T07:09:34.802904+0000 mon.a (mon.0) 650 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:09:36.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:09:35 smithi089 bash[82202]: audit 2024-07-15T07:09:34.802904+0000 mon.a (mon.0) 650 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:09:37.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:09:36 smithi134 bash[67624]: cluster 2024-07-15T07:09:35.003052+0000 mgr.y (mgr.25244) 468 : cluster [DBG] pgmap v413: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:09:37.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:09:36 smithi089 bash[79810]: cluster 2024-07-15T07:09:35.003052+0000 mgr.y (mgr.25244) 468 : cluster [DBG] pgmap v413: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:09:37.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:09:36 smithi089 bash[82202]: cluster 2024-07-15T07:09:35.003052+0000 mgr.y (mgr.25244) 468 : cluster [DBG] pgmap v413: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:09:39.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:09:38 smithi134 bash[67624]: cluster 2024-07-15T07:09:37.004140+0000 mgr.y (mgr.25244) 469 : cluster [DBG] pgmap v414: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:09:39.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:09:38 smithi089 bash[79810]: cluster 2024-07-15T07:09:37.004140+0000 mgr.y (mgr.25244) 469 : cluster [DBG] pgmap v414: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:09:39.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:09:38 smithi089 bash[82202]: cluster 2024-07-15T07:09:37.004140+0000 mgr.y (mgr.25244) 469 : cluster [DBG] pgmap v414: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:09:39.246 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T07:09:41.057 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:09:40 smithi134 bash[67624]: cluster 2024-07-15T07:09:39.004998+0000 mgr.y (mgr.25244) 470 : cluster [DBG] pgmap v415: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:09:41.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:09:40 smithi089 bash[79810]: cluster 2024-07-15T07:09:39.004998+0000 mgr.y (mgr.25244) 470 : cluster [DBG] pgmap v415: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:09:41.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:09:40 smithi089 bash[82202]: cluster 2024-07-15T07:09:39.004998+0000 mgr.y (mgr.25244) 470 : cluster [DBG] pgmap v415: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:09:42.738 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:09:42 smithi134 bash[60729]: ts=2024-07-15T07:09:42.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T07:09:42.804 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-07-15T07:09:42.804 INFO:teuthology.orchestra.run.smithi089.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-07-15T07:09:43.056 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:09:42 smithi089 bash[82202]: cluster 2024-07-15T07:09:41.006110+0000 mgr.y (mgr.25244) 471 : cluster [DBG] pgmap v416: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:09:43.056 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:09:42 smithi089 bash[79810]: cluster 2024-07-15T07:09:41.006110+0000 mgr.y (mgr.25244) 471 : cluster [DBG] pgmap v416: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:09:43.057 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:09:42 smithi134 bash[67624]: cluster 2024-07-15T07:09:41.006110+0000 mgr.y (mgr.25244) 471 : cluster [DBG] pgmap v416: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:09:43.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:09:43 smithi089 bash[79810]: audit 2024-07-15T07:09:42.807432+0000 mon.a (mon.0) 651 : audit [DBG] from='client.? 172.21.15.89:0/3918200965' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-15T07:09:43.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:09:43 smithi089 bash[82202]: audit 2024-07-15T07:09:42.807432+0000 mon.a (mon.0) 651 : audit [DBG] from='client.? 172.21.15.89:0/3918200965' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-15T07:09:44.057 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:09:43 smithi134 bash[67624]: audit 2024-07-15T07:09:42.807432+0000 mon.a (mon.0) 651 : audit [DBG] from='client.? 172.21.15.89:0/3918200965' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-15T07:09:44.622 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 07:09:44 smithi089 bash[72894]: ::ffff:172.21.15.134 - - [15/Jul/2024:07:09:44] "GET /metrics HTTP/1.1" 200 33975 "" "Prometheus/2.43.0" 2024-07-15T07:09:45.057 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:09:44 smithi134 bash[67624]: cluster 2024-07-15T07:09:43.006868+0000 mgr.y (mgr.25244) 472 : cluster [DBG] pgmap v417: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:09:45.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:09:44 smithi089 bash[79810]: cluster 2024-07-15T07:09:43.006868+0000 mgr.y (mgr.25244) 472 : cluster [DBG] pgmap v417: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:09:45.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:09:44 smithi089 bash[82202]: cluster 2024-07-15T07:09:43.006868+0000 mgr.y (mgr.25244) 472 : cluster [DBG] pgmap v417: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:09:46.574 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph health --format=json 2024-07-15T07:09:46.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:09:46 smithi089 bash[79810]: cluster 2024-07-15T07:09:45.007666+0000 mgr.y (mgr.25244) 473 : cluster [DBG] pgmap v418: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:09:46.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:09:46 smithi089 bash[82202]: cluster 2024-07-15T07:09:45.007666+0000 mgr.y (mgr.25244) 473 : cluster [DBG] pgmap v418: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:09:47.057 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:09:46 smithi134 bash[67624]: cluster 2024-07-15T07:09:45.007666+0000 mgr.y (mgr.25244) 473 : cluster [DBG] pgmap v418: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:09:49.057 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:09:48 smithi134 bash[67624]: cluster 2024-07-15T07:09:47.008561+0000 mgr.y (mgr.25244) 474 : cluster [DBG] pgmap v419: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:09:49.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:09:48 smithi089 bash[79810]: cluster 2024-07-15T07:09:47.008561+0000 mgr.y (mgr.25244) 474 : cluster [DBG] pgmap v419: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:09:49.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:09:48 smithi089 bash[82202]: cluster 2024-07-15T07:09:47.008561+0000 mgr.y (mgr.25244) 474 : cluster [DBG] pgmap v419: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:09:51.057 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:09:50 smithi134 bash[67624]: cluster 2024-07-15T07:09:49.009287+0000 mgr.y (mgr.25244) 475 : cluster [DBG] pgmap v420: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:09:51.057 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:09:50 smithi134 bash[67624]: audit 2024-07-15T07:09:49.803367+0000 mon.a (mon.0) 652 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:09:51.057 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:09:50 smithi134 bash[67624]: audit 2024-07-15T07:09:50.232397+0000 mon.a (mon.0) 653 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T07:09:51.057 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:09:50 smithi134 bash[67624]: audit 2024-07-15T07:09:50.603433+0000 mon.a (mon.0) 654 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:09:51.057 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:09:50 smithi134 bash[67624]: audit 2024-07-15T07:09:50.608384+0000 mon.a (mon.0) 655 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:09:51.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:09:50 smithi089 bash[82202]: cluster 2024-07-15T07:09:49.009287+0000 mgr.y (mgr.25244) 475 : cluster [DBG] pgmap v420: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:09:51.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:09:50 smithi089 bash[82202]: audit 2024-07-15T07:09:49.803367+0000 mon.a (mon.0) 652 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:09:51.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:09:50 smithi089 bash[82202]: audit 2024-07-15T07:09:50.232397+0000 mon.a (mon.0) 653 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T07:09:51.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:09:50 smithi089 bash[82202]: audit 2024-07-15T07:09:50.603433+0000 mon.a (mon.0) 654 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:09:51.123 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:09:50 smithi089 bash[82202]: audit 2024-07-15T07:09:50.608384+0000 mon.a (mon.0) 655 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:09:51.123 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:09:50 smithi089 bash[79810]: cluster 2024-07-15T07:09:49.009287+0000 mgr.y (mgr.25244) 475 : cluster [DBG] pgmap v420: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:09:51.123 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:09:50 smithi089 bash[79810]: audit 2024-07-15T07:09:49.803367+0000 mon.a (mon.0) 652 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:09:51.123 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:09:50 smithi089 bash[79810]: audit 2024-07-15T07:09:50.232397+0000 mon.a (mon.0) 653 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T07:09:51.123 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:09:50 smithi089 bash[79810]: audit 2024-07-15T07:09:50.603433+0000 mon.a (mon.0) 654 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:09:51.123 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:09:50 smithi089 bash[79810]: audit 2024-07-15T07:09:50.608384+0000 mon.a (mon.0) 655 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:09:51.364 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T07:09:52.765 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:09:52 smithi134 bash[60729]: ts=2024-07-15T07:09:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T07:09:52.987 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:09:52 smithi089 bash[79810]: cluster 2024-07-15T07:09:51.010394+0000 mgr.y (mgr.25244) 476 : cluster [DBG] pgmap v421: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:09:52.987 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:09:52 smithi089 bash[82202]: cluster 2024-07-15T07:09:51.010394+0000 mgr.y (mgr.25244) 476 : cluster [DBG] pgmap v421: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:09:53.057 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:09:52 smithi134 bash[67624]: cluster 2024-07-15T07:09:51.010394+0000 mgr.y (mgr.25244) 476 : cluster [DBG] pgmap v421: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:09:54.620 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 07:09:54 smithi089 bash[72894]: ::ffff:172.21.15.134 - - [15/Jul/2024:07:09:54] "GET /metrics HTTP/1.1" 200 33976 "" "Prometheus/2.43.0" 2024-07-15T07:09:54.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:09:54 smithi089 bash[79810]: cluster 2024-07-15T07:09:53.011188+0000 mgr.y (mgr.25244) 477 : cluster [DBG] pgmap v422: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:09:54.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:09:54 smithi089 bash[82202]: cluster 2024-07-15T07:09:53.011188+0000 mgr.y (mgr.25244) 477 : cluster [DBG] pgmap v422: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:09:55.034 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-07-15T07:09:55.034 INFO:teuthology.orchestra.run.smithi089.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-07-15T07:09:55.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:09:54 smithi134 bash[67624]: cluster 2024-07-15T07:09:53.011188+0000 mgr.y (mgr.25244) 477 : cluster [DBG] pgmap v422: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:09:56.056 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:09:55 smithi134 bash[67624]: audit 2024-07-15T07:09:55.037039+0000 mon.a (mon.0) 656 : audit [DBG] from='client.? 172.21.15.89:0/425978040' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-15T07:09:56.057 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:09:55 smithi134 bash[67624]: audit 2024-07-15T07:09:55.102133+0000 mon.a (mon.0) 657 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:09:56.057 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:09:55 smithi134 bash[67624]: audit 2024-07-15T07:09:55.108263+0000 mon.a (mon.0) 658 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:09:56.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:09:55 smithi089 bash[79810]: audit 2024-07-15T07:09:55.037039+0000 mon.a (mon.0) 656 : audit [DBG] from='client.? 172.21.15.89:0/425978040' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-15T07:09:56.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:09:55 smithi089 bash[79810]: audit 2024-07-15T07:09:55.102133+0000 mon.a (mon.0) 657 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:09:56.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:09:55 smithi089 bash[79810]: audit 2024-07-15T07:09:55.108263+0000 mon.a (mon.0) 658 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:09:56.123 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:09:55 smithi089 bash[82202]: audit 2024-07-15T07:09:55.037039+0000 mon.a (mon.0) 656 : audit [DBG] from='client.? 172.21.15.89:0/425978040' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-15T07:09:56.123 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:09:55 smithi089 bash[82202]: audit 2024-07-15T07:09:55.102133+0000 mon.a (mon.0) 657 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:09:56.123 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:09:55 smithi089 bash[82202]: audit 2024-07-15T07:09:55.108263+0000 mon.a (mon.0) 658 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:09:57.057 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:09:56 smithi134 bash[67624]: cluster 2024-07-15T07:09:55.011684+0000 mgr.y (mgr.25244) 478 : cluster [DBG] pgmap v423: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:09:57.057 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:09:56 smithi134 bash[67624]: audit 2024-07-15T07:09:56.401015+0000 mon.a (mon.0) 659 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:09:57.057 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:09:56 smithi134 bash[67624]: audit 2024-07-15T07:09:56.411500+0000 mon.a (mon.0) 660 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:09:57.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:09:56 smithi089 bash[79810]: cluster 2024-07-15T07:09:55.011684+0000 mgr.y (mgr.25244) 478 : cluster [DBG] pgmap v423: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:09:57.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:09:56 smithi089 bash[79810]: audit 2024-07-15T07:09:56.401015+0000 mon.a (mon.0) 659 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:09:57.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:09:56 smithi089 bash[79810]: audit 2024-07-15T07:09:56.411500+0000 mon.a (mon.0) 660 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:09:57.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:09:56 smithi089 bash[82202]: cluster 2024-07-15T07:09:55.011684+0000 mgr.y (mgr.25244) 478 : cluster [DBG] pgmap v423: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:09:57.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:09:56 smithi089 bash[82202]: audit 2024-07-15T07:09:56.401015+0000 mon.a (mon.0) 659 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:09:57.123 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:09:56 smithi089 bash[82202]: audit 2024-07-15T07:09:56.411500+0000 mon.a (mon.0) 660 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:09:58.057 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:09:57 smithi134 bash[67624]: audit 2024-07-15T07:09:56.880880+0000 mon.a (mon.0) 661 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T07:09:58.057 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:09:57 smithi134 bash[67624]: audit 2024-07-15T07:09:56.882786+0000 mon.a (mon.0) 662 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T07:09:58.057 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:09:57 smithi134 bash[67624]: audit 2024-07-15T07:09:56.894982+0000 mon.a (mon.0) 663 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:09:58.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:09:57 smithi089 bash[79810]: audit 2024-07-15T07:09:56.880880+0000 mon.a (mon.0) 661 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T07:09:58.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:09:57 smithi089 bash[79810]: audit 2024-07-15T07:09:56.882786+0000 mon.a (mon.0) 662 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T07:09:58.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:09:57 smithi089 bash[79810]: audit 2024-07-15T07:09:56.894982+0000 mon.a (mon.0) 663 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:09:58.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:09:57 smithi089 bash[82202]: audit 2024-07-15T07:09:56.880880+0000 mon.a (mon.0) 661 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T07:09:58.123 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:09:57 smithi089 bash[82202]: audit 2024-07-15T07:09:56.882786+0000 mon.a (mon.0) 662 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T07:09:58.123 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:09:57 smithi089 bash[82202]: audit 2024-07-15T07:09:56.894982+0000 mon.a (mon.0) 663 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:09:58.795 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph health --format=json 2024-07-15T07:09:59.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:09:58 smithi089 bash[79810]: cluster 2024-07-15T07:09:57.012616+0000 mgr.y (mgr.25244) 479 : cluster [DBG] pgmap v424: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:09:59.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:09:58 smithi089 bash[82202]: cluster 2024-07-15T07:09:57.012616+0000 mgr.y (mgr.25244) 479 : cluster [DBG] pgmap v424: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:09:59.307 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:09:58 smithi134 bash[67624]: cluster 2024-07-15T07:09:57.012616+0000 mgr.y (mgr.25244) 479 : cluster [DBG] pgmap v424: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:10:00.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:10:00 smithi089 bash[79810]: debug 2024-07-15T07:09:59.995+0000 7fb563a0a640 -1 log_channel(cluster) log [ERR] : overall HEALTH_ERR 8 osds(s) are not reachable 2024-07-15T07:10:01.307 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:10:00 smithi134 bash[67624]: cluster 2024-07-15T07:09:59.013365+0000 mgr.y (mgr.25244) 480 : cluster [DBG] pgmap v425: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:10:01.307 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:10:00 smithi134 bash[67624]: cluster 2024-07-15T07:10:00.000124+0000 mon.a (mon.0) 664 : cluster [ERR] overall HEALTH_ERR 8 osds(s) are not reachable 2024-07-15T07:10:01.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:10:00 smithi089 bash[79810]: cluster 2024-07-15T07:09:59.013365+0000 mgr.y (mgr.25244) 480 : cluster [DBG] pgmap v425: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:10:01.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:10:00 smithi089 bash[79810]: cluster 2024-07-15T07:10:00.000124+0000 mon.a (mon.0) 664 : cluster [ERR] overall HEALTH_ERR 8 osds(s) are not reachable 2024-07-15T07:10:01.372 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:10:00 smithi089 bash[82202]: cluster 2024-07-15T07:09:59.013365+0000 mgr.y (mgr.25244) 480 : cluster [DBG] pgmap v425: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:10:01.372 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:10:00 smithi089 bash[82202]: cluster 2024-07-15T07:10:00.000124+0000 mon.a (mon.0) 664 : cluster [ERR] overall HEALTH_ERR 8 osds(s) are not reachable 2024-07-15T07:10:02.807 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:10:02 smithi134 bash[60729]: ts=2024-07-15T07:10:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T07:10:03.307 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:10:02 smithi134 bash[67624]: cluster 2024-07-15T07:10:01.014333+0000 mgr.y (mgr.25244) 481 : cluster [DBG] pgmap v426: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:10:03.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:10:02 smithi089 bash[79810]: cluster 2024-07-15T07:10:01.014333+0000 mgr.y (mgr.25244) 481 : cluster [DBG] pgmap v426: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:10:03.372 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:10:02 smithi089 bash[82202]: cluster 2024-07-15T07:10:01.014333+0000 mgr.y (mgr.25244) 481 : cluster [DBG] pgmap v426: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:10:03.580 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T07:10:04.622 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 07:10:04 smithi089 bash[72894]: ::ffff:172.21.15.134 - - [15/Jul/2024:07:10:04] "GET /metrics HTTP/1.1" 200 33976 "" "Prometheus/2.43.0" 2024-07-15T07:10:05.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:10:04 smithi089 bash[79810]: cluster 2024-07-15T07:10:03.015124+0000 mgr.y (mgr.25244) 482 : cluster [DBG] pgmap v427: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:10:05.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:10:04 smithi089 bash[79810]: audit 2024-07-15T07:10:04.804075+0000 mon.a (mon.0) 665 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:10:05.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:10:04 smithi089 bash[82202]: cluster 2024-07-15T07:10:03.015124+0000 mgr.y (mgr.25244) 482 : cluster [DBG] pgmap v427: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:10:05.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:10:04 smithi089 bash[82202]: audit 2024-07-15T07:10:04.804075+0000 mon.a (mon.0) 665 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:10:05.307 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:10:04 smithi134 bash[67624]: cluster 2024-07-15T07:10:03.015124+0000 mgr.y (mgr.25244) 482 : cluster [DBG] pgmap v427: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:10:05.307 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:10:04 smithi134 bash[67624]: audit 2024-07-15T07:10:04.804075+0000 mon.a (mon.0) 665 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:10:07.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:10:06 smithi089 bash[79810]: cluster 2024-07-15T07:10:05.015757+0000 mgr.y (mgr.25244) 483 : cluster [DBG] pgmap v428: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:10:07.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:10:06 smithi089 bash[82202]: cluster 2024-07-15T07:10:05.015757+0000 mgr.y (mgr.25244) 483 : cluster [DBG] pgmap v428: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:10:07.191 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-07-15T07:10:07.191 INFO:teuthology.orchestra.run.smithi089.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-07-15T07:10:07.307 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:10:06 smithi134 bash[67624]: cluster 2024-07-15T07:10:05.015757+0000 mgr.y (mgr.25244) 483 : cluster [DBG] pgmap v428: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:10:08.307 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:10:07 smithi134 bash[67624]: audit 2024-07-15T07:10:07.194564+0000 mon.c (mon.2) 30 : audit [DBG] from='client.? 172.21.15.89:0/3975665408' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-15T07:10:08.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:10:07 smithi089 bash[79810]: audit 2024-07-15T07:10:07.194564+0000 mon.c (mon.2) 30 : audit [DBG] from='client.? 172.21.15.89:0/3975665408' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-15T07:10:08.372 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:10:07 smithi089 bash[82202]: audit 2024-07-15T07:10:07.194564+0000 mon.c (mon.2) 30 : audit [DBG] from='client.? 172.21.15.89:0/3975665408' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-15T07:10:09.307 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:10:08 smithi134 bash[67624]: cluster 2024-07-15T07:10:07.016516+0000 mgr.y (mgr.25244) 484 : cluster [DBG] pgmap v429: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:10:09.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:10:08 smithi089 bash[79810]: cluster 2024-07-15T07:10:07.016516+0000 mgr.y (mgr.25244) 484 : cluster [DBG] pgmap v429: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:10:09.372 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:10:08 smithi089 bash[82202]: cluster 2024-07-15T07:10:07.016516+0000 mgr.y (mgr.25244) 484 : cluster [DBG] pgmap v429: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:10:10.307 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:10:09 smithi134 bash[67624]: cluster 2024-07-15T07:10:09.017221+0000 mgr.y (mgr.25244) 485 : cluster [DBG] pgmap v430: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:10:10.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:10:09 smithi089 bash[79810]: cluster 2024-07-15T07:10:09.017221+0000 mgr.y (mgr.25244) 485 : cluster [DBG] pgmap v430: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:10:10.372 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:10:09 smithi089 bash[82202]: cluster 2024-07-15T07:10:09.017221+0000 mgr.y (mgr.25244) 485 : cluster [DBG] pgmap v430: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:10:10.987 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph health --format=json 2024-07-15T07:10:12.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:10:12 smithi089 bash[79810]: cluster 2024-07-15T07:10:11.018220+0000 mgr.y (mgr.25244) 486 : cluster [DBG] pgmap v431: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:10:12.372 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:10:12 smithi089 bash[82202]: cluster 2024-07-15T07:10:11.018220+0000 mgr.y (mgr.25244) 486 : cluster [DBG] pgmap v431: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:10:12.557 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:10:12 smithi134 bash[67624]: cluster 2024-07-15T07:10:11.018220+0000 mgr.y (mgr.25244) 486 : cluster [DBG] pgmap v431: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:10:12.557 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:10:12 smithi134 bash[60729]: ts=2024-07-15T07:10:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T07:10:14.422 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 07:10:14 smithi089 bash[72894]: ::ffff:172.21.15.134 - - [15/Jul/2024:07:10:14] "GET /metrics HTTP/1.1" 200 33977 "" "Prometheus/2.43.0" 2024-07-15T07:10:14.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:10:14 smithi134 bash[67624]: cluster 2024-07-15T07:10:13.018950+0000 mgr.y (mgr.25244) 487 : cluster [DBG] pgmap v432: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:10:14.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:10:14 smithi089 bash[82202]: cluster 2024-07-15T07:10:13.018950+0000 mgr.y (mgr.25244) 487 : cluster [DBG] pgmap v432: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:10:14.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:10:14 smithi089 bash[79810]: cluster 2024-07-15T07:10:13.018950+0000 mgr.y (mgr.25244) 487 : cluster [DBG] pgmap v432: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:10:15.773 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T07:10:16.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:10:16 smithi134 bash[67624]: cluster 2024-07-15T07:10:15.019777+0000 mgr.y (mgr.25244) 488 : cluster [DBG] pgmap v433: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:10:16.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:10:16 smithi089 bash[79810]: cluster 2024-07-15T07:10:15.019777+0000 mgr.y (mgr.25244) 488 : cluster [DBG] pgmap v433: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:10:16.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:10:16 smithi089 bash[82202]: cluster 2024-07-15T07:10:15.019777+0000 mgr.y (mgr.25244) 488 : cluster [DBG] pgmap v433: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:10:18.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:10:18 smithi134 bash[67624]: cluster 2024-07-15T07:10:17.020481+0000 mgr.y (mgr.25244) 489 : cluster [DBG] pgmap v434: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:10:18.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:10:18 smithi089 bash[79810]: cluster 2024-07-15T07:10:17.020481+0000 mgr.y (mgr.25244) 489 : cluster [DBG] pgmap v434: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:10:18.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:10:18 smithi089 bash[82202]: cluster 2024-07-15T07:10:17.020481+0000 mgr.y (mgr.25244) 489 : cluster [DBG] pgmap v434: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:10:19.444 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-07-15T07:10:19.444 INFO:teuthology.orchestra.run.smithi089.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-07-15T07:10:20.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:10:20 smithi089 bash[82202]: cluster 2024-07-15T07:10:19.021300+0000 mgr.y (mgr.25244) 490 : cluster [DBG] pgmap v435: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:10:20.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:10:20 smithi089 bash[82202]: audit 2024-07-15T07:10:19.446771+0000 mon.c (mon.2) 31 : audit [DBG] from='client.? 172.21.15.89:0/1622414654' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-15T07:10:20.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:10:20 smithi089 bash[82202]: audit 2024-07-15T07:10:19.804113+0000 mon.a (mon.0) 666 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:10:20.622 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:10:20 smithi089 bash[79810]: cluster 2024-07-15T07:10:19.021300+0000 mgr.y (mgr.25244) 490 : cluster [DBG] pgmap v435: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:10:20.623 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:10:20 smithi089 bash[79810]: audit 2024-07-15T07:10:19.446771+0000 mon.c (mon.2) 31 : audit [DBG] from='client.? 172.21.15.89:0/1622414654' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-15T07:10:20.623 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:10:20 smithi089 bash[79810]: audit 2024-07-15T07:10:19.804113+0000 mon.a (mon.0) 666 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:10:20.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:10:20 smithi134 bash[67624]: cluster 2024-07-15T07:10:19.021300+0000 mgr.y (mgr.25244) 490 : cluster [DBG] pgmap v435: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:10:20.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:10:20 smithi134 bash[67624]: audit 2024-07-15T07:10:19.446771+0000 mon.c (mon.2) 31 : audit [DBG] from='client.? 172.21.15.89:0/1622414654' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-15T07:10:20.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:10:20 smithi134 bash[67624]: audit 2024-07-15T07:10:19.804113+0000 mon.a (mon.0) 666 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:10:22.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:10:22 smithi134 bash[67624]: cluster 2024-07-15T07:10:21.022289+0000 mgr.y (mgr.25244) 491 : cluster [DBG] pgmap v436: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:10:22.807 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:10:22 smithi134 bash[60729]: ts=2024-07-15T07:10:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T07:10:22.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:10:22 smithi089 bash[79810]: cluster 2024-07-15T07:10:21.022289+0000 mgr.y (mgr.25244) 491 : cluster [DBG] pgmap v436: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:10:22.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:10:22 smithi089 bash[82202]: cluster 2024-07-15T07:10:21.022289+0000 mgr.y (mgr.25244) 491 : cluster [DBG] pgmap v436: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:10:23.239 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph health --format=json 2024-07-15T07:10:24.442 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 07:10:24 smithi089 bash[72894]: ::ffff:172.21.15.134 - - [15/Jul/2024:07:10:24] "GET /metrics HTTP/1.1" 200 33976 "" "Prometheus/2.43.0" 2024-07-15T07:10:24.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:10:24 smithi134 bash[67624]: cluster 2024-07-15T07:10:23.023003+0000 mgr.y (mgr.25244) 492 : cluster [DBG] pgmap v437: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:10:24.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:10:24 smithi089 bash[79810]: cluster 2024-07-15T07:10:23.023003+0000 mgr.y (mgr.25244) 492 : cluster [DBG] pgmap v437: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:10:24.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:10:24 smithi089 bash[82202]: cluster 2024-07-15T07:10:23.023003+0000 mgr.y (mgr.25244) 492 : cluster [DBG] pgmap v437: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:10:26.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:10:26 smithi134 bash[67624]: cluster 2024-07-15T07:10:25.023829+0000 mgr.y (mgr.25244) 493 : cluster [DBG] pgmap v438: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:10:26.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:10:26 smithi089 bash[79810]: cluster 2024-07-15T07:10:25.023829+0000 mgr.y (mgr.25244) 493 : cluster [DBG] pgmap v438: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:10:26.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:10:26 smithi089 bash[82202]: cluster 2024-07-15T07:10:25.023829+0000 mgr.y (mgr.25244) 493 : cluster [DBG] pgmap v438: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:10:28.022 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T07:10:28.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:10:28 smithi134 bash[67624]: cluster 2024-07-15T07:10:27.024779+0000 mgr.y (mgr.25244) 494 : cluster [DBG] pgmap v439: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:10:28.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:10:28 smithi089 bash[79810]: cluster 2024-07-15T07:10:27.024779+0000 mgr.y (mgr.25244) 494 : cluster [DBG] pgmap v439: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:10:28.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:10:28 smithi089 bash[82202]: cluster 2024-07-15T07:10:27.024779+0000 mgr.y (mgr.25244) 494 : cluster [DBG] pgmap v439: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:10:30.622 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:10:30 smithi089 bash[79810]: cluster 2024-07-15T07:10:29.025473+0000 mgr.y (mgr.25244) 495 : cluster [DBG] pgmap v440: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:10:30.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:10:30 smithi089 bash[82202]: cluster 2024-07-15T07:10:29.025473+0000 mgr.y (mgr.25244) 495 : cluster [DBG] pgmap v440: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:10:30.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:10:30 smithi134 bash[67624]: cluster 2024-07-15T07:10:29.025473+0000 mgr.y (mgr.25244) 495 : cluster [DBG] pgmap v440: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:10:31.742 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-07-15T07:10:31.743 INFO:teuthology.orchestra.run.smithi089.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-07-15T07:10:32.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:10:32 smithi134 bash[67624]: cluster 2024-07-15T07:10:31.026474+0000 mgr.y (mgr.25244) 496 : cluster [DBG] pgmap v441: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:10:32.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:10:32 smithi134 bash[67624]: audit 2024-07-15T07:10:31.745618+0000 mon.c (mon.2) 32 : audit [DBG] from='client.? 172.21.15.89:0/3894770274' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-15T07:10:32.808 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:10:32 smithi134 bash[60729]: ts=2024-07-15T07:10:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T07:10:32.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:10:32 smithi089 bash[79810]: cluster 2024-07-15T07:10:31.026474+0000 mgr.y (mgr.25244) 496 : cluster [DBG] pgmap v441: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:10:32.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:10:32 smithi089 bash[79810]: audit 2024-07-15T07:10:31.745618+0000 mon.c (mon.2) 32 : audit [DBG] from='client.? 172.21.15.89:0/3894770274' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-15T07:10:32.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:10:32 smithi089 bash[82202]: cluster 2024-07-15T07:10:31.026474+0000 mgr.y (mgr.25244) 496 : cluster [DBG] pgmap v441: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:10:32.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:10:32 smithi089 bash[82202]: audit 2024-07-15T07:10:31.745618+0000 mon.c (mon.2) 32 : audit [DBG] from='client.? 172.21.15.89:0/3894770274' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-15T07:10:34.464 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 07:10:34 smithi089 bash[72894]: ::ffff:172.21.15.134 - - [15/Jul/2024:07:10:34] "GET /metrics HTTP/1.1" 200 33976 "" "Prometheus/2.43.0" 2024-07-15T07:10:34.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:10:34 smithi134 bash[67624]: cluster 2024-07-15T07:10:33.027179+0000 mgr.y (mgr.25244) 497 : cluster [DBG] pgmap v442: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:10:34.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:10:34 smithi089 bash[79810]: cluster 2024-07-15T07:10:33.027179+0000 mgr.y (mgr.25244) 497 : cluster [DBG] pgmap v442: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:10:34.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:10:34 smithi089 bash[82202]: cluster 2024-07-15T07:10:33.027179+0000 mgr.y (mgr.25244) 497 : cluster [DBG] pgmap v442: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:10:35.467 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph health --format=json 2024-07-15T07:10:35.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:10:35 smithi134 bash[67624]: audit 2024-07-15T07:10:34.804507+0000 mon.a (mon.0) 667 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:10:35.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:10:35 smithi089 bash[79810]: audit 2024-07-15T07:10:34.804507+0000 mon.a (mon.0) 667 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:10:35.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:10:35 smithi089 bash[82202]: audit 2024-07-15T07:10:34.804507+0000 mon.a (mon.0) 667 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:10:36.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:10:36 smithi134 bash[67624]: cluster 2024-07-15T07:10:35.027964+0000 mgr.y (mgr.25244) 498 : cluster [DBG] pgmap v443: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:10:36.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:10:36 smithi089 bash[79810]: cluster 2024-07-15T07:10:35.027964+0000 mgr.y (mgr.25244) 498 : cluster [DBG] pgmap v443: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:10:36.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:10:36 smithi089 bash[82202]: cluster 2024-07-15T07:10:35.027964+0000 mgr.y (mgr.25244) 498 : cluster [DBG] pgmap v443: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:10:38.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:10:38 smithi134 bash[67624]: cluster 2024-07-15T07:10:37.029023+0000 mgr.y (mgr.25244) 499 : cluster [DBG] pgmap v444: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:10:38.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:10:38 smithi089 bash[79810]: cluster 2024-07-15T07:10:37.029023+0000 mgr.y (mgr.25244) 499 : cluster [DBG] pgmap v444: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:10:38.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:10:38 smithi089 bash[82202]: cluster 2024-07-15T07:10:37.029023+0000 mgr.y (mgr.25244) 499 : cluster [DBG] pgmap v444: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:10:40.253 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T07:10:40.622 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:10:40 smithi089 bash[79810]: cluster 2024-07-15T07:10:39.029904+0000 mgr.y (mgr.25244) 500 : cluster [DBG] pgmap v445: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:10:40.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:10:40 smithi089 bash[82202]: cluster 2024-07-15T07:10:39.029904+0000 mgr.y (mgr.25244) 500 : cluster [DBG] pgmap v445: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:10:40.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:10:40 smithi134 bash[67624]: cluster 2024-07-15T07:10:39.029904+0000 mgr.y (mgr.25244) 500 : cluster [DBG] pgmap v445: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:10:41.307 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 07:10:40 smithi134 bash[59462]: logger=sqlstore.transactions t=2024-07-15T07:10:40.919108207Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-07-15T07:10:42.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:10:42 smithi134 bash[67624]: cluster 2024-07-15T07:10:41.030931+0000 mgr.y (mgr.25244) 501 : cluster [DBG] pgmap v446: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:10:42.808 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:10:42 smithi134 bash[60729]: ts=2024-07-15T07:10:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T07:10:42.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:10:42 smithi089 bash[79810]: cluster 2024-07-15T07:10:41.030931+0000 mgr.y (mgr.25244) 501 : cluster [DBG] pgmap v446: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:10:42.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:10:42 smithi089 bash[82202]: cluster 2024-07-15T07:10:41.030931+0000 mgr.y (mgr.25244) 501 : cluster [DBG] pgmap v446: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:10:44.038 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-07-15T07:10:44.038 INFO:teuthology.orchestra.run.smithi089.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-07-15T07:10:44.372 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 07:10:44 smithi089 bash[72894]: ::ffff:172.21.15.134 - - [15/Jul/2024:07:10:44] "GET /metrics HTTP/1.1" 200 33978 "" "Prometheus/2.43.0" 2024-07-15T07:10:44.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:10:44 smithi134 bash[67624]: cluster 2024-07-15T07:10:43.031660+0000 mgr.y (mgr.25244) 502 : cluster [DBG] pgmap v447: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:10:44.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:10:44 smithi134 bash[67624]: audit 2024-07-15T07:10:44.040886+0000 mon.a (mon.0) 668 : audit [DBG] from='client.? 172.21.15.89:0/3843027143' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-15T07:10:44.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:10:44 smithi089 bash[79810]: cluster 2024-07-15T07:10:43.031660+0000 mgr.y (mgr.25244) 502 : cluster [DBG] pgmap v447: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:10:44.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:10:44 smithi089 bash[79810]: audit 2024-07-15T07:10:44.040886+0000 mon.a (mon.0) 668 : audit [DBG] from='client.? 172.21.15.89:0/3843027143' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-15T07:10:44.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:10:44 smithi089 bash[82202]: cluster 2024-07-15T07:10:43.031660+0000 mgr.y (mgr.25244) 502 : cluster [DBG] pgmap v447: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:10:44.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:10:44 smithi089 bash[82202]: audit 2024-07-15T07:10:44.040886+0000 mon.a (mon.0) 668 : audit [DBG] from='client.? 172.21.15.89:0/3843027143' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-15T07:10:46.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:10:46 smithi134 bash[67624]: cluster 2024-07-15T07:10:45.032556+0000 mgr.y (mgr.25244) 503 : cluster [DBG] pgmap v448: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:10:46.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:10:46 smithi089 bash[79810]: cluster 2024-07-15T07:10:45.032556+0000 mgr.y (mgr.25244) 503 : cluster [DBG] pgmap v448: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:10:46.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:10:46 smithi089 bash[82202]: cluster 2024-07-15T07:10:45.032556+0000 mgr.y (mgr.25244) 503 : cluster [DBG] pgmap v448: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:10:47.703 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph health --format=json 2024-07-15T07:10:48.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:10:48 smithi134 bash[67624]: cluster 2024-07-15T07:10:47.033631+0000 mgr.y (mgr.25244) 504 : cluster [DBG] pgmap v449: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:10:48.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:10:48 smithi089 bash[79810]: cluster 2024-07-15T07:10:47.033631+0000 mgr.y (mgr.25244) 504 : cluster [DBG] pgmap v449: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:10:48.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:10:48 smithi089 bash[82202]: cluster 2024-07-15T07:10:47.033631+0000 mgr.y (mgr.25244) 504 : cluster [DBG] pgmap v449: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:10:50.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:10:50 smithi134 bash[67624]: cluster 2024-07-15T07:10:49.034395+0000 mgr.y (mgr.25244) 505 : cluster [DBG] pgmap v450: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:10:50.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:10:50 smithi134 bash[67624]: audit 2024-07-15T07:10:49.804964+0000 mon.a (mon.0) 669 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:10:50.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:10:50 smithi089 bash[79810]: cluster 2024-07-15T07:10:49.034395+0000 mgr.y (mgr.25244) 505 : cluster [DBG] pgmap v450: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:10:50.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:10:50 smithi089 bash[79810]: audit 2024-07-15T07:10:49.804964+0000 mon.a (mon.0) 669 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:10:50.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:10:50 smithi089 bash[82202]: cluster 2024-07-15T07:10:49.034395+0000 mgr.y (mgr.25244) 505 : cluster [DBG] pgmap v450: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:10:50.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:10:50 smithi089 bash[82202]: audit 2024-07-15T07:10:49.804964+0000 mon.a (mon.0) 669 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:10:52.491 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T07:10:52.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:10:52 smithi134 bash[67624]: cluster 2024-07-15T07:10:51.035537+0000 mgr.y (mgr.25244) 506 : cluster [DBG] pgmap v451: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:10:52.807 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:10:52 smithi134 bash[60729]: ts=2024-07-15T07:10:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T07:10:52.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:10:52 smithi089 bash[79810]: cluster 2024-07-15T07:10:51.035537+0000 mgr.y (mgr.25244) 506 : cluster [DBG] pgmap v451: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:10:52.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:10:52 smithi089 bash[82202]: cluster 2024-07-15T07:10:51.035537+0000 mgr.y (mgr.25244) 506 : cluster [DBG] pgmap v451: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:10:54.372 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 07:10:54 smithi089 bash[72894]: ::ffff:172.21.15.134 - - [15/Jul/2024:07:10:54] "GET /metrics HTTP/1.1" 200 33978 "" "Prometheus/2.43.0" 2024-07-15T07:10:54.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:10:54 smithi134 bash[67624]: cluster 2024-07-15T07:10:53.036368+0000 mgr.y (mgr.25244) 507 : cluster [DBG] pgmap v452: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:10:54.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:10:54 smithi089 bash[79810]: cluster 2024-07-15T07:10:53.036368+0000 mgr.y (mgr.25244) 507 : cluster [DBG] pgmap v452: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:10:54.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:10:54 smithi089 bash[82202]: cluster 2024-07-15T07:10:53.036368+0000 mgr.y (mgr.25244) 507 : cluster [DBG] pgmap v452: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:10:56.124 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-07-15T07:10:56.125 INFO:teuthology.orchestra.run.smithi089.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-07-15T07:10:56.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:10:56 smithi134 bash[67624]: cluster 2024-07-15T07:10:55.037163+0000 mgr.y (mgr.25244) 508 : cluster [DBG] pgmap v453: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:10:56.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:10:56 smithi134 bash[67624]: audit 2024-07-15T07:10:56.127735+0000 mon.a (mon.0) 670 : audit [DBG] from='client.? 172.21.15.89:0/2518908010' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-15T07:10:56.862 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:10:56 smithi089 bash[79810]: cluster 2024-07-15T07:10:55.037163+0000 mgr.y (mgr.25244) 508 : cluster [DBG] pgmap v453: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:10:56.862 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:10:56 smithi089 bash[79810]: audit 2024-07-15T07:10:56.127735+0000 mon.a (mon.0) 670 : audit [DBG] from='client.? 172.21.15.89:0/2518908010' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-15T07:10:56.863 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:10:56 smithi089 bash[82202]: cluster 2024-07-15T07:10:55.037163+0000 mgr.y (mgr.25244) 508 : cluster [DBG] pgmap v453: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:10:56.863 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:10:56 smithi089 bash[82202]: audit 2024-07-15T07:10:56.127735+0000 mon.a (mon.0) 670 : audit [DBG] from='client.? 172.21.15.89:0/2518908010' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-15T07:10:57.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:10:57 smithi134 bash[67624]: audit 2024-07-15T07:10:57.027187+0000 mon.a (mon.0) 671 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T07:10:57.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:10:57 smithi089 bash[79810]: audit 2024-07-15T07:10:57.027187+0000 mon.a (mon.0) 671 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T07:10:57.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:10:57 smithi089 bash[82202]: audit 2024-07-15T07:10:57.027187+0000 mon.a (mon.0) 671 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-15T07:10:58.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:10:58 smithi134 bash[67624]: cluster 2024-07-15T07:10:57.038116+0000 mgr.y (mgr.25244) 509 : cluster [DBG] pgmap v454: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:10:58.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:10:58 smithi089 bash[82202]: cluster 2024-07-15T07:10:57.038116+0000 mgr.y (mgr.25244) 509 : cluster [DBG] pgmap v454: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:10:58.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:10:58 smithi089 bash[79810]: cluster 2024-07-15T07:10:57.038116+0000 mgr.y (mgr.25244) 509 : cluster [DBG] pgmap v454: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:10:59.865 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 -- ceph health --format=json 2024-07-15T07:11:00.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:11:00 smithi134 bash[67624]: cluster 2024-07-15T07:10:59.038842+0000 mgr.y (mgr.25244) 510 : cluster [DBG] pgmap v455: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:11:00.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:11:00 smithi089 bash[79810]: cluster 2024-07-15T07:10:59.038842+0000 mgr.y (mgr.25244) 510 : cluster [DBG] pgmap v455: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:11:00.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:11:00 smithi089 bash[82202]: cluster 2024-07-15T07:10:59.038842+0000 mgr.y (mgr.25244) 510 : cluster [DBG] pgmap v455: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:11:02.807 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:11:02 smithi134 bash[67624]: cluster 2024-07-15T07:11:01.039915+0000 mgr.y (mgr.25244) 511 : cluster [DBG] pgmap v456: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:11:02.807 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:11:02 smithi134 bash[60729]: ts=2024-07-15T07:11:02.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T07:11:02.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:11:02 smithi089 bash[79810]: cluster 2024-07-15T07:11:01.039915+0000 mgr.y (mgr.25244) 511 : cluster [DBG] pgmap v456: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:11:02.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:11:02 smithi089 bash[82202]: cluster 2024-07-15T07:11:01.039915+0000 mgr.y (mgr.25244) 511 : cluster [DBG] pgmap v456: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:11:03.970 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/mon.a/config 2024-07-15T07:11:04.052 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:11:03 smithi089 bash[82202]: audit 2024-07-15T07:11:02.637750+0000 mon.a (mon.0) 672 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:11:04.052 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:11:03 smithi089 bash[82202]: audit 2024-07-15T07:11:02.645177+0000 mon.a (mon.0) 673 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:11:04.052 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:11:03 smithi089 bash[82202]: audit 2024-07-15T07:11:02.883210+0000 mon.a (mon.0) 674 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:11:04.052 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:11:03 smithi089 bash[82202]: audit 2024-07-15T07:11:02.895829+0000 mon.a (mon.0) 675 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:11:04.052 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:11:03 smithi089 bash[82202]: audit 2024-07-15T07:11:03.309177+0000 mon.a (mon.0) 676 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T07:11:04.052 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:11:03 smithi089 bash[82202]: audit 2024-07-15T07:11:03.310933+0000 mon.a (mon.0) 677 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T07:11:04.053 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:11:03 smithi089 bash[82202]: audit 2024-07-15T07:11:03.324979+0000 mon.a (mon.0) 678 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:11:04.053 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:11:03 smithi089 bash[79810]: audit 2024-07-15T07:11:02.637750+0000 mon.a (mon.0) 672 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:11:04.053 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:11:03 smithi089 bash[79810]: audit 2024-07-15T07:11:02.645177+0000 mon.a (mon.0) 673 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:11:04.053 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:11:03 smithi089 bash[79810]: audit 2024-07-15T07:11:02.883210+0000 mon.a (mon.0) 674 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:11:04.054 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:11:03 smithi089 bash[79810]: audit 2024-07-15T07:11:02.895829+0000 mon.a (mon.0) 675 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:11:04.054 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:11:03 smithi089 bash[79810]: audit 2024-07-15T07:11:03.309177+0000 mon.a (mon.0) 676 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T07:11:04.054 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:11:03 smithi089 bash[79810]: audit 2024-07-15T07:11:03.310933+0000 mon.a (mon.0) 677 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T07:11:04.054 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:11:03 smithi089 bash[79810]: audit 2024-07-15T07:11:03.324979+0000 mon.a (mon.0) 678 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:11:04.057 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:11:03 smithi134 bash[67624]: audit 2024-07-15T07:11:02.637750+0000 mon.a (mon.0) 672 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:11:04.057 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:11:03 smithi134 bash[67624]: audit 2024-07-15T07:11:02.645177+0000 mon.a (mon.0) 673 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:11:04.057 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:11:03 smithi134 bash[67624]: audit 2024-07-15T07:11:02.883210+0000 mon.a (mon.0) 674 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:11:04.058 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:11:03 smithi134 bash[67624]: audit 2024-07-15T07:11:02.895829+0000 mon.a (mon.0) 675 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:11:04.058 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:11:03 smithi134 bash[67624]: audit 2024-07-15T07:11:03.309177+0000 mon.a (mon.0) 676 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-15T07:11:04.058 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:11:03 smithi134 bash[67624]: audit 2024-07-15T07:11:03.310933+0000 mon.a (mon.0) 677 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-15T07:11:04.058 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:11:03 smithi134 bash[67624]: audit 2024-07-15T07:11:03.324979+0000 mon.a (mon.0) 678 : audit [INF] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' 2024-07-15T07:11:04.371 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 07:11:04 smithi089 bash[72894]: ::ffff:172.21.15.134 - - [15/Jul/2024:07:11:04] "GET /metrics HTTP/1.1" 200 33978 "" "Prometheus/2.43.0" 2024-07-15T07:11:05.236 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:11:04 smithi089 bash[79810]: cluster 2024-07-15T07:11:03.040380+0000 mgr.y (mgr.25244) 512 : cluster [DBG] pgmap v457: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:11:05.236 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:11:04 smithi089 bash[79810]: audit 2024-07-15T07:11:04.806318+0000 mon.a (mon.0) 679 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:11:05.237 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:11:04 smithi089 bash[82202]: cluster 2024-07-15T07:11:03.040380+0000 mgr.y (mgr.25244) 512 : cluster [DBG] pgmap v457: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:11:05.237 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:11:04 smithi089 bash[82202]: audit 2024-07-15T07:11:04.806318+0000 mon.a (mon.0) 679 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:11:05.307 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:11:04 smithi134 bash[67624]: cluster 2024-07-15T07:11:03.040380+0000 mgr.y (mgr.25244) 512 : cluster [DBG] pgmap v457: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:11:05.307 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:11:04 smithi134 bash[67624]: audit 2024-07-15T07:11:04.806318+0000 mon.a (mon.0) 679 : audit [DBG] from='mgr.25244 172.21.15.89:0/913358296' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-15T07:11:07.179 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:11:06 smithi089 bash[79810]: cluster 2024-07-15T07:11:05.041166+0000 mgr.y (mgr.25244) 513 : cluster [DBG] pgmap v458: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:11:07.179 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:11:06 smithi089 bash[82202]: cluster 2024-07-15T07:11:05.041166+0000 mgr.y (mgr.25244) 513 : cluster [DBG] pgmap v458: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:11:07.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:11:06 smithi134 bash[67624]: cluster 2024-07-15T07:11:05.041166+0000 mgr.y (mgr.25244) 513 : cluster [DBG] pgmap v458: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-07-15T07:11:07.535 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-07-15T07:11:07.535 INFO:teuthology.orchestra.run.smithi089.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-07-15T07:11:08.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:11:07 smithi089 bash[79810]: audit 2024-07-15T07:11:07.538442+0000 mon.a (mon.0) 680 : audit [DBG] from='client.? 172.21.15.89:0/4294135617' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-15T07:11:08.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:11:07 smithi089 bash[82202]: audit 2024-07-15T07:11:07.538442+0000 mon.a (mon.0) 680 : audit [DBG] from='client.? 172.21.15.89:0/4294135617' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-15T07:11:08.305 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_454a46106b61e201598ae5e1c392af2328539bfe/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_454a46106b61e201598ae5e1c392af2328539bfe/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_9122136bebef94311801796793b3ccd81a4a9765/qa/tasks/ceph.py", line 1494, in healthy manager.wait_until_healthy(timeout=300) File "/home/teuthworker/src/git.ceph.com_ceph_9122136bebef94311801796793b3ccd81a4a9765/qa/tasks/ceph_manager.py", line 3191, in wait_until_healthy assert time.time() - start < timeout, \ AssertionError: timeout expired in wait_until_healthy 2024-07-15T07:11:08.526 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=76429c7c47aa429a8711dcf795f3cb96 Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_454a46106b61e201598ae5e1c392af2328539bfe/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_454a46106b61e201598ae5e1c392af2328539bfe/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_9122136bebef94311801796793b3ccd81a4a9765/qa/tasks/ceph.py", line 1494, in healthy manager.wait_until_healthy(timeout=300) File "/home/teuthworker/src/git.ceph.com_ceph_9122136bebef94311801796793b3ccd81a4a9765/qa/tasks/ceph_manager.py", line 3191, in wait_until_healthy assert time.time() - start < timeout, \ AssertionError: timeout expired in wait_until_healthy 2024-07-15T07:11:08.528 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-07-15T07:11:08.537 INFO:tasks.cephadm:Teardown begin 2024-07-15T07:11:08.537 DEBUG:teuthology.orchestra.run.smithi089:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-07-15T07:11:08.541 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:11:07 smithi134 bash[67624]: audit 2024-07-15T07:11:07.538442+0000 mon.a (mon.0) 680 : audit [DBG] from='client.? 172.21.15.89:0/4294135617' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-15T07:11:08.553 DEBUG:teuthology.orchestra.run.smithi134:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-07-15T07:11:08.568 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-07-15T07:11:08.568 DEBUG:teuthology.orchestra.run.smithi089:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-07-15T07:11:08.600 DEBUG:teuthology.orchestra.run.smithi134:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-07-15T07:11:08.615 INFO:tasks.cephadm:Stopping all daemons... 2024-07-15T07:11:08.615 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2024-07-15T07:11:08.615 DEBUG:teuthology.orchestra.run.smithi089:> sudo systemctl stop ceph-75ba02bc-4274-11ef-bcad-c7b262605968@mon.a 2024-07-15T07:11:08.955 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 07:11:08 smithi089 bash[72894]: [15/Jul/2024:07:11:08] ENGINE Bus STOPPING 2024-07-15T07:11:08.956 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:11:08 smithi089 systemd[1]: Stopping Ceph mon.a for 75ba02bc-4274-11ef-bcad-c7b262605968... 2024-07-15T07:11:08.956 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:11:08 smithi089 bash[79810]: debug 2024-07-15T07:11:08.778+0000 7fb566a10640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2024-07-15T07:11:08.956 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:11:08 smithi089 bash[79810]: debug 2024-07-15T07:11:08.778+0000 7fb566a10640 -1 mon.a@0(leader) e4 *** Got Signal Terminated *** 2024-07-15T07:11:09.271 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 07:11:09 smithi089 bash[72894]: [15/Jul/2024:07:11:09] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-07-15T07:11:09.272 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 07:11:09 smithi089 bash[72894]: [15/Jul/2024:07:11:09] ENGINE Bus STOPPED 2024-07-15T07:11:09.272 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 07:11:09 smithi089 bash[72894]: [15/Jul/2024:07:11:09] ENGINE Bus STARTING 2024-07-15T07:11:09.614 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 07:11:09 smithi089 bash[72894]: [15/Jul/2024:07:11:09] ENGINE Serving on http://:::9283 2024-07-15T07:11:09.614 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 07:11:09 smithi089 bash[72894]: [15/Jul/2024:07:11:09] ENGINE Bus STARTED 2024-07-15T07:11:09.856 DEBUG:teuthology.orchestra.run.smithi089:> sudo pkill -f 'journalctl -f -n 0 -u ceph-75ba02bc-4274-11ef-bcad-c7b262605968@mon.a.service' 2024-07-15T07:11:09.865 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:11:09 smithi089 bash[118371]: ceph-75ba02bc-4274-11ef-bcad-c7b262605968-mon-a 2024-07-15T07:11:09.865 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:11:09 smithi089 bash[118443]: Error response from daemon: No such container: ceph-75ba02bc-4274-11ef-bcad-c7b262605968-mon-a 2024-07-15T07:11:09.865 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:11:09 smithi089 systemd[1]: ceph-75ba02bc-4274-11ef-bcad-c7b262605968@mon.a.service: Deactivated successfully. 2024-07-15T07:11:09.866 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jul 15 07:11:09 smithi089 systemd[1]: Stopped Ceph mon.a for 75ba02bc-4274-11ef-bcad-c7b262605968. 2024-07-15T07:11:09.892 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-07-15T07:11:09.893 INFO:tasks.cephadm.mon.a:Stopped mon.a 2024-07-15T07:11:09.893 INFO:tasks.cephadm.mon.b:Stopping mon.c... 2024-07-15T07:11:09.893 DEBUG:teuthology.orchestra.run.smithi089:> sudo systemctl stop ceph-75ba02bc-4274-11ef-bcad-c7b262605968@mon.c 2024-07-15T07:11:10.121 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 07:11:10 smithi089 bash[72894]: [15/Jul/2024:07:11:10] ENGINE Bus STOPPING 2024-07-15T07:11:10.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:11:09 smithi089 systemd[1]: Stopping Ceph mon.c for 75ba02bc-4274-11ef-bcad-c7b262605968... 2024-07-15T07:11:10.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:11:10 smithi089 bash[82202]: debug 2024-07-15T07:11:10.074+0000 7f4f36847640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2024-07-15T07:11:10.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:11:10 smithi089 bash[82202]: debug 2024-07-15T07:11:10.074+0000 7f4f36847640 -1 mon.c@2(peon) e4 *** Got Signal Terminated *** 2024-07-15T07:11:10.622 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 07:11:10 smithi089 bash[72894]: [15/Jul/2024:07:11:10] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-07-15T07:11:10.622 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 07:11:10 smithi089 bash[72894]: [15/Jul/2024:07:11:10] ENGINE Bus STOPPED 2024-07-15T07:11:10.622 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 07:11:10 smithi089 bash[72894]: [15/Jul/2024:07:11:10] ENGINE Bus STARTING 2024-07-15T07:11:10.622 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 07:11:10 smithi089 bash[72894]: [15/Jul/2024:07:11:10] ENGINE Serving on http://:::9283 2024-07-15T07:11:10.622 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 07:11:10 smithi089 bash[72894]: [15/Jul/2024:07:11:10] ENGINE Bus STARTED 2024-07-15T07:11:11.052 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jul 15 07:11:10 smithi089 bash[118482]: ceph-75ba02bc-4274-11ef-bcad-c7b262605968-mon-c 2024-07-15T07:11:11.057 DEBUG:teuthology.orchestra.run.smithi089:> sudo pkill -f 'journalctl -f -n 0 -u ceph-75ba02bc-4274-11ef-bcad-c7b262605968@mon.c.service' 2024-07-15T07:11:11.081 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-07-15T07:11:11.082 INFO:tasks.cephadm.mon.b:Stopped mon.c 2024-07-15T07:11:11.082 INFO:tasks.cephadm.mon.b:Stopping mon.b... 2024-07-15T07:11:11.082 DEBUG:teuthology.orchestra.run.smithi134:> sudo systemctl stop ceph-75ba02bc-4274-11ef-bcad-c7b262605968@mon.b 2024-07-15T07:11:11.473 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:11:11 smithi134 systemd[1]: Stopping Ceph mon.b for 75ba02bc-4274-11ef-bcad-c7b262605968... 2024-07-15T07:11:11.473 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:11:11 smithi134 bash[67624]: debug 2024-07-15T07:11:11.230+0000 7f2234119640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2024-07-15T07:11:11.473 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:11:11 smithi134 bash[67624]: debug 2024-07-15T07:11:11.230+0000 7f2234119640 -1 mon.b@1(peon) e4 *** Got Signal Terminated *** 2024-07-15T07:11:12.382 DEBUG:teuthology.orchestra.run.smithi134:> sudo pkill -f 'journalctl -f -n 0 -u ceph-75ba02bc-4274-11ef-bcad-c7b262605968@mon.b.service' 2024-07-15T07:11:12.391 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:11:12 smithi134 bash[60729]: ts=2024-07-15T07:11:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"75ba02bc-4274-11ef-bcad-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-15T07:11:12.391 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:11:12 smithi134 bash[79221]: ceph-75ba02bc-4274-11ef-bcad-c7b262605968-mon-b 2024-07-15T07:11:12.391 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:11:12 smithi134 bash[79290]: Error response from daemon: No such container: ceph-75ba02bc-4274-11ef-bcad-c7b262605968-mon-b 2024-07-15T07:11:12.391 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:11:12 smithi134 systemd[1]: ceph-75ba02bc-4274-11ef-bcad-c7b262605968@mon.b.service: Deactivated successfully. 2024-07-15T07:11:12.392 INFO:journalctl@ceph.mon.b.smithi134.stdout:Jul 15 07:11:12 smithi134 systemd[1]: Stopped Ceph mon.b for 75ba02bc-4274-11ef-bcad-c7b262605968. 2024-07-15T07:11:12.418 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-07-15T07:11:12.418 INFO:tasks.cephadm.mon.b:Stopped mon.b 2024-07-15T07:11:12.418 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2024-07-15T07:11:12.419 DEBUG:teuthology.orchestra.run.smithi089:> sudo systemctl stop ceph-75ba02bc-4274-11ef-bcad-c7b262605968@mgr.y 2024-07-15T07:11:12.872 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 07:11:12 smithi089 systemd[1]: Stopping Ceph mgr.y for 75ba02bc-4274-11ef-bcad-c7b262605968... 2024-07-15T07:11:12.872 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 07:11:12 smithi089 bash[72894]: debug 2024-07-15T07:11:12.574+0000 7f1181e1e640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mgr -n mgr.y -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-07-15T07:11:12.872 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 07:11:12 smithi089 bash[72894]: debug 2024-07-15T07:11:12.574+0000 7f1181e1e640 -1 mgr handle_mgr_signal *** Got signal Terminated *** 2024-07-15T07:11:13.515 DEBUG:teuthology.orchestra.run.smithi089:> sudo pkill -f 'journalctl -f -n 0 -u ceph-75ba02bc-4274-11ef-bcad-c7b262605968@mgr.y.service' 2024-07-15T07:11:13.524 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 07:11:13 smithi089 bash[118592]: ceph-75ba02bc-4274-11ef-bcad-c7b262605968-mgr-y 2024-07-15T07:11:13.524 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 07:11:13 smithi089 bash[118652]: Error response from daemon: No such container: ceph-75ba02bc-4274-11ef-bcad-c7b262605968-mgr-y 2024-07-15T07:11:13.524 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 07:11:13 smithi089 systemd[1]: ceph-75ba02bc-4274-11ef-bcad-c7b262605968@mgr.y.service: Deactivated successfully. 2024-07-15T07:11:13.524 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jul 15 07:11:13 smithi089 systemd[1]: Stopped Ceph mgr.y for 75ba02bc-4274-11ef-bcad-c7b262605968. 2024-07-15T07:11:13.538 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-07-15T07:11:13.539 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2024-07-15T07:11:13.539 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2024-07-15T07:11:13.539 DEBUG:teuthology.orchestra.run.smithi134:> sudo systemctl stop ceph-75ba02bc-4274-11ef-bcad-c7b262605968@mgr.x 2024-07-15T07:11:13.807 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 07:11:13 smithi134 systemd[1]: Stopping Ceph mgr.x for 75ba02bc-4274-11ef-bcad-c7b262605968... 2024-07-15T07:11:14.583 DEBUG:teuthology.orchestra.run.smithi134:> sudo pkill -f 'journalctl -f -n 0 -u ceph-75ba02bc-4274-11ef-bcad-c7b262605968@mgr.x.service' 2024-07-15T07:11:14.592 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 07:11:14 smithi134 bash[79329]: ceph-75ba02bc-4274-11ef-bcad-c7b262605968-mgr-x 2024-07-15T07:11:14.592 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 07:11:14 smithi134 systemd[1]: ceph-75ba02bc-4274-11ef-bcad-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-07-15T07:11:14.592 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 07:11:14 smithi134 bash[79391]: Error response from daemon: No such container: ceph-75ba02bc-4274-11ef-bcad-c7b262605968-mgr-x 2024-07-15T07:11:14.592 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 07:11:14 smithi134 systemd[1]: ceph-75ba02bc-4274-11ef-bcad-c7b262605968@mgr.x.service: Failed with result 'exit-code'. 2024-07-15T07:11:14.593 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Jul 15 07:11:14 smithi134 systemd[1]: Stopped Ceph mgr.x for 75ba02bc-4274-11ef-bcad-c7b262605968. 2024-07-15T07:11:14.606 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-07-15T07:11:14.606 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2024-07-15T07:11:14.606 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-07-15T07:11:14.606 DEBUG:teuthology.orchestra.run.smithi089:> sudo systemctl stop ceph-75ba02bc-4274-11ef-bcad-c7b262605968@osd.0 2024-07-15T07:11:14.872 INFO:journalctl@ceph.osd.0.smithi089.stdout:Jul 15 07:11:14 smithi089 systemd[1]: Stopping Ceph osd.0 for 75ba02bc-4274-11ef-bcad-c7b262605968... 2024-07-15T07:11:14.872 INFO:journalctl@ceph.osd.0.smithi089.stdout:Jul 15 07:11:14 smithi089 bash[88569]: debug 2024-07-15T07:11:14.750+0000 7f165bd11640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-07-15T07:11:14.872 INFO:journalctl@ceph.osd.0.smithi089.stdout:Jul 15 07:11:14 smithi089 bash[88569]: debug 2024-07-15T07:11:14.750+0000 7f165bd11640 -1 osd.0 114 *** Got signal Terminated *** 2024-07-15T07:11:14.872 INFO:journalctl@ceph.osd.0.smithi089.stdout:Jul 15 07:11:14 smithi089 bash[88569]: debug 2024-07-15T07:11:14.750+0000 7f165bd11640 -1 osd.0 114 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-07-15T07:11:20.621 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 07:11:20 smithi089 bash[72306]: ts=2024-07-15T07:11:20.175Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi089.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.89:8443: connect: connection refused" 2024-07-15T07:11:20.622 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 07:11:20 smithi089 bash[72306]: ts=2024-07-15T07:11:20.176Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi134.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.134:8443: connect: connection refused" 2024-07-15T07:11:21.121 INFO:journalctl@ceph.osd.0.smithi089.stdout:Jul 15 07:11:20 smithi089 bash[118691]: ceph-75ba02bc-4274-11ef-bcad-c7b262605968-osd-0 2024-07-15T07:11:21.122 INFO:journalctl@ceph.osd.0.smithi089.stdout:Jul 15 07:11:20 smithi089 bash[118756]: Error response from daemon: No such container: ceph-75ba02bc-4274-11ef-bcad-c7b262605968-osd-0 2024-07-15T07:11:22.895 DEBUG:teuthology.orchestra.run.smithi089:> sudo pkill -f 'journalctl -f -n 0 -u ceph-75ba02bc-4274-11ef-bcad-c7b262605968@osd.0.service' 2024-07-15T07:11:22.921 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-07-15T07:11:22.921 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-07-15T07:11:22.922 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-07-15T07:11:22.922 DEBUG:teuthology.orchestra.run.smithi089:> sudo systemctl stop ceph-75ba02bc-4274-11ef-bcad-c7b262605968@osd.1 2024-07-15T07:11:23.371 INFO:journalctl@ceph.osd.1.smithi089.stdout:Jul 15 07:11:22 smithi089 systemd[1]: Stopping Ceph osd.1 for 75ba02bc-4274-11ef-bcad-c7b262605968... 2024-07-15T07:11:23.372 INFO:journalctl@ceph.osd.1.smithi089.stdout:Jul 15 07:11:23 smithi089 bash[93078]: debug 2024-07-15T07:11:23.102+0000 7f7bf5e54640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-07-15T07:11:23.372 INFO:journalctl@ceph.osd.1.smithi089.stdout:Jul 15 07:11:23 smithi089 bash[93078]: debug 2024-07-15T07:11:23.102+0000 7f7bf5e54640 -1 osd.1 114 *** Got signal Terminated *** 2024-07-15T07:11:23.372 INFO:journalctl@ceph.osd.1.smithi089.stdout:Jul 15 07:11:23 smithi089 bash[93078]: debug 2024-07-15T07:11:23.102+0000 7f7bf5e54640 -1 osd.1 114 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-07-15T07:11:28.058 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:11:27 smithi134 bash[60729]: ts=2024-07-15T07:11:27.713Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.89:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.89:8765: connect: connection refused" 2024-07-15T07:11:28.058 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:11:27 smithi134 bash[60729]: ts=2024-07-15T07:11:27.715Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.89:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.89:8765: connect: connection refused" 2024-07-15T07:11:28.058 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:11:27 smithi134 bash[60729]: ts=2024-07-15T07:11:27.720Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=nvmeof msg="Unable to refresh target groups" err="Get \"http://172.21.15.89:8765/sd/prometheus/sd-config?service=nvmeof\": dial tcp 172.21.15.89:8765: connect: connection refused" 2024-07-15T07:11:28.058 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:11:27 smithi134 bash[60729]: ts=2024-07-15T07:11:27.722Z caller=refresh.go:99 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.89:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.89:8765: connect: connection refused" 2024-07-15T07:11:28.059 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:11:27 smithi134 bash[60729]: ts=2024-07-15T07:11:27.722Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.89:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.89:8765: connect: connection refused" 2024-07-15T07:11:29.621 INFO:journalctl@ceph.osd.1.smithi089.stdout:Jul 15 07:11:29 smithi089 bash[118885]: ceph-75ba02bc-4274-11ef-bcad-c7b262605968-osd-1 2024-07-15T07:11:29.622 INFO:journalctl@ceph.osd.1.smithi089.stdout:Jul 15 07:11:29 smithi089 bash[118949]: Error response from daemon: No such container: ceph-75ba02bc-4274-11ef-bcad-c7b262605968-osd-1 2024-07-15T07:11:30.575 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 07:11:30 smithi089 bash[72306]: ts=2024-07-15T07:11:30.175Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi134.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.134:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi089.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.89:8443: connect: connection refused" 2024-07-15T07:11:30.575 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 07:11:30 smithi089 bash[72306]: ts=2024-07-15T07:11:30.176Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi089.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.89:8443: connect: connection refused" 2024-07-15T07:11:30.575 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 07:11:30 smithi089 bash[72306]: ts=2024-07-15T07:11:30.178Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi134.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.134:8443: connect: connection refused" 2024-07-15T07:11:31.573 DEBUG:teuthology.orchestra.run.smithi089:> sudo pkill -f 'journalctl -f -n 0 -u ceph-75ba02bc-4274-11ef-bcad-c7b262605968@osd.1.service' 2024-07-15T07:11:31.601 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-07-15T07:11:31.602 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-07-15T07:11:31.602 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-07-15T07:11:31.602 DEBUG:teuthology.orchestra.run.smithi089:> sudo systemctl stop ceph-75ba02bc-4274-11ef-bcad-c7b262605968@osd.2 2024-07-15T07:11:31.872 INFO:journalctl@ceph.osd.2.smithi089.stdout:Jul 15 07:11:31 smithi089 systemd[1]: Stopping Ceph osd.2 for 75ba02bc-4274-11ef-bcad-c7b262605968... 2024-07-15T07:11:31.872 INFO:journalctl@ceph.osd.2.smithi089.stdout:Jul 15 07:11:31 smithi089 bash[98398]: debug 2024-07-15T07:11:31.782+0000 7f0f7d8cf640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-07-15T07:11:31.872 INFO:journalctl@ceph.osd.2.smithi089.stdout:Jul 15 07:11:31 smithi089 bash[98398]: debug 2024-07-15T07:11:31.782+0000 7f0f7d8cf640 -1 osd.2 114 *** Got signal Terminated *** 2024-07-15T07:11:31.872 INFO:journalctl@ceph.osd.2.smithi089.stdout:Jul 15 07:11:31 smithi089 bash[98398]: debug 2024-07-15T07:11:31.782+0000 7f0f7d8cf640 -1 osd.2 114 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-07-15T07:11:38.058 INFO:journalctl@ceph.osd.2.smithi089.stdout:Jul 15 07:11:37 smithi089 bash[119076]: ceph-75ba02bc-4274-11ef-bcad-c7b262605968-osd-2 2024-07-15T07:11:38.059 INFO:journalctl@ceph.osd.2.smithi089.stdout:Jul 15 07:11:37 smithi089 bash[119142]: Error response from daemon: No such container: ceph-75ba02bc-4274-11ef-bcad-c7b262605968-osd-2 2024-07-15T07:11:39.846 DEBUG:teuthology.orchestra.run.smithi089:> sudo pkill -f 'journalctl -f -n 0 -u ceph-75ba02bc-4274-11ef-bcad-c7b262605968@osd.2.service' 2024-07-15T07:11:39.874 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-07-15T07:11:39.874 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-07-15T07:11:39.875 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-07-15T07:11:39.875 DEBUG:teuthology.orchestra.run.smithi089:> sudo systemctl stop ceph-75ba02bc-4274-11ef-bcad-c7b262605968@osd.3 2024-07-15T07:11:40.121 INFO:journalctl@ceph.osd.3.smithi089.stdout:Jul 15 07:11:39 smithi089 systemd[1]: Stopping Ceph osd.3 for 75ba02bc-4274-11ef-bcad-c7b262605968... 2024-07-15T07:11:40.122 INFO:journalctl@ceph.osd.3.smithi089.stdout:Jul 15 07:11:40 smithi089 bash[102706]: debug 2024-07-15T07:11:40.057+0000 7f46f5105640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-07-15T07:11:40.122 INFO:journalctl@ceph.osd.3.smithi089.stdout:Jul 15 07:11:40 smithi089 bash[102706]: debug 2024-07-15T07:11:40.057+0000 7f46f5105640 -1 osd.3 114 *** Got signal Terminated *** 2024-07-15T07:11:40.122 INFO:journalctl@ceph.osd.3.smithi089.stdout:Jul 15 07:11:40 smithi089 bash[102706]: debug 2024-07-15T07:11:40.057+0000 7f46f5105640 -1 osd.3 114 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-07-15T07:11:40.621 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 07:11:40 smithi089 bash[72306]: ts=2024-07-15T07:11:40.175Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi089.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.89:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi134.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.134:8443: connect: connection refused" 2024-07-15T07:11:40.622 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 07:11:40 smithi089 bash[72306]: ts=2024-07-15T07:11:40.176Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi089.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.89:8443: connect: connection refused" 2024-07-15T07:11:40.622 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 07:11:40 smithi089 bash[72306]: ts=2024-07-15T07:11:40.178Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi134.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.134:8443: connect: connection refused" 2024-07-15T07:11:41.308 INFO:journalctl@ceph.osd.5.smithi134.stdout:Jul 15 07:11:40 smithi134 bash[33230]: debug 2024-07-15T07:11:40.889+0000 7f19b4257640 -1 osd.5 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:16.769056+0000 front 2024-07-15T07:11:16.769321+0000 (oldest deadline 2024-07-15T07:11:40.269036+0000) 2024-07-15T07:11:42.299 INFO:journalctl@ceph.osd.5.smithi134.stdout:Jul 15 07:11:41 smithi134 bash[33230]: debug 2024-07-15T07:11:41.861+0000 7f19b4257640 -1 osd.5 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:16.769056+0000 front 2024-07-15T07:11:16.769321+0000 (oldest deadline 2024-07-15T07:11:40.269036+0000) 2024-07-15T07:11:42.299 INFO:journalctl@ceph.osd.6.smithi134.stdout:Jul 15 07:11:41 smithi134 bash[38841]: debug 2024-07-15T07:11:41.845+0000 7fb73c9d7640 -1 osd.6 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:15.147034+0000 front 2024-07-15T07:11:15.147232+0000 (oldest deadline 2024-07-15T07:11:41.046849+0000) 2024-07-15T07:11:42.557 INFO:journalctl@ceph.osd.4.smithi134.stdout:Jul 15 07:11:42 smithi134 bash[73271]: debug 2024-07-15T07:11:42.293+0000 7fcf66132640 -1 osd.4 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:18.715966+0000 front 2024-07-15T07:11:18.715810+0000 (oldest deadline 2024-07-15T07:11:42.116030+0000) 2024-07-15T07:11:43.289 INFO:journalctl@ceph.osd.5.smithi134.stdout:Jul 15 07:11:42 smithi134 bash[33230]: debug 2024-07-15T07:11:42.865+0000 7f19b4257640 -1 osd.5 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:16.769056+0000 front 2024-07-15T07:11:16.769321+0000 (oldest deadline 2024-07-15T07:11:40.269036+0000) 2024-07-15T07:11:43.289 INFO:journalctl@ceph.osd.6.smithi134.stdout:Jul 15 07:11:42 smithi134 bash[38841]: debug 2024-07-15T07:11:42.833+0000 7fb73c9d7640 -1 osd.6 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:15.147034+0000 front 2024-07-15T07:11:15.147232+0000 (oldest deadline 2024-07-15T07:11:41.046849+0000) 2024-07-15T07:11:43.558 INFO:journalctl@ceph.osd.4.smithi134.stdout:Jul 15 07:11:43 smithi134 bash[73271]: debug 2024-07-15T07:11:43.289+0000 7fcf66132640 -1 osd.4 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:18.715966+0000 front 2024-07-15T07:11:18.715810+0000 (oldest deadline 2024-07-15T07:11:42.116030+0000) 2024-07-15T07:11:43.871 INFO:journalctl@ceph.osd.3.smithi089.stdout:Jul 15 07:11:43 smithi089 bash[102706]: debug 2024-07-15T07:11:43.489+0000 7f46ed711640 -1 osd.3 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:17.579503+0000 front 2024-07-15T07:11:17.579558+0000 (oldest deadline 2024-07-15T07:11:43.380408+0000) 2024-07-15T07:11:44.276 INFO:journalctl@ceph.osd.5.smithi134.stdout:Jul 15 07:11:43 smithi134 bash[33230]: debug 2024-07-15T07:11:43.873+0000 7f19b4257640 -1 osd.5 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:16.769056+0000 front 2024-07-15T07:11:16.769321+0000 (oldest deadline 2024-07-15T07:11:40.269036+0000) 2024-07-15T07:11:44.277 INFO:journalctl@ceph.osd.6.smithi134.stdout:Jul 15 07:11:43 smithi134 bash[38841]: debug 2024-07-15T07:11:43.833+0000 7fb73c9d7640 -1 osd.6 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:15.147034+0000 front 2024-07-15T07:11:15.147232+0000 (oldest deadline 2024-07-15T07:11:41.046849+0000) 2024-07-15T07:11:44.277 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:11:43 smithi134 bash[44248]: debug 2024-07-15T07:11:43.989+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:19.694655+0000 front 2024-07-15T07:11:19.695037+0000 (oldest deadline 2024-07-15T07:11:43.794595+0000) 2024-07-15T07:11:44.557 INFO:journalctl@ceph.osd.4.smithi134.stdout:Jul 15 07:11:44 smithi134 bash[73271]: debug 2024-07-15T07:11:44.273+0000 7fcf66132640 -1 osd.4 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:18.715966+0000 front 2024-07-15T07:11:18.715810+0000 (oldest deadline 2024-07-15T07:11:42.116030+0000) 2024-07-15T07:11:44.871 INFO:journalctl@ceph.osd.3.smithi089.stdout:Jul 15 07:11:44 smithi089 bash[102706]: debug 2024-07-15T07:11:44.449+0000 7f46ed711640 -1 osd.3 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:17.579503+0000 front 2024-07-15T07:11:17.579558+0000 (oldest deadline 2024-07-15T07:11:43.380408+0000) 2024-07-15T07:11:45.262 INFO:journalctl@ceph.osd.5.smithi134.stdout:Jul 15 07:11:44 smithi134 bash[33230]: debug 2024-07-15T07:11:44.913+0000 7f19b4257640 -1 osd.5 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:16.769056+0000 front 2024-07-15T07:11:16.769321+0000 (oldest deadline 2024-07-15T07:11:40.269036+0000) 2024-07-15T07:11:45.263 INFO:journalctl@ceph.osd.6.smithi134.stdout:Jul 15 07:11:44 smithi134 bash[38841]: debug 2024-07-15T07:11:44.849+0000 7fb73c9d7640 -1 osd.6 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:15.147034+0000 front 2024-07-15T07:11:15.147232+0000 (oldest deadline 2024-07-15T07:11:41.046849+0000) 2024-07-15T07:11:45.263 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:11:44 smithi134 bash[44248]: debug 2024-07-15T07:11:44.945+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:19.694655+0000 front 2024-07-15T07:11:19.695037+0000 (oldest deadline 2024-07-15T07:11:43.794595+0000) 2024-07-15T07:11:45.557 INFO:journalctl@ceph.osd.4.smithi134.stdout:Jul 15 07:11:45 smithi134 bash[73271]: debug 2024-07-15T07:11:45.257+0000 7fcf66132640 -1 osd.4 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:18.715966+0000 front 2024-07-15T07:11:18.715810+0000 (oldest deadline 2024-07-15T07:11:42.116030+0000) 2024-07-15T07:11:46.222 INFO:journalctl@ceph.osd.6.smithi134.stdout:Jul 15 07:11:45 smithi134 bash[38841]: debug 2024-07-15T07:11:45.849+0000 7fb73c9d7640 -1 osd.6 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:15.147034+0000 front 2024-07-15T07:11:15.147232+0000 (oldest deadline 2024-07-15T07:11:41.046849+0000) 2024-07-15T07:11:46.222 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:11:45 smithi134 bash[44248]: debug 2024-07-15T07:11:45.945+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:19.694655+0000 front 2024-07-15T07:11:19.695037+0000 (oldest deadline 2024-07-15T07:11:43.794595+0000) 2024-07-15T07:11:46.223 INFO:journalctl@ceph.osd.5.smithi134.stdout:Jul 15 07:11:45 smithi134 bash[33230]: debug 2024-07-15T07:11:45.937+0000 7f19b4257640 -1 osd.5 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:16.769056+0000 front 2024-07-15T07:11:16.769321+0000 (oldest deadline 2024-07-15T07:11:40.269036+0000) 2024-07-15T07:11:46.371 INFO:journalctl@ceph.osd.3.smithi089.stdout:Jul 15 07:11:46 smithi089 bash[119270]: ceph-75ba02bc-4274-11ef-bcad-c7b262605968-osd-3 2024-07-15T07:11:46.371 INFO:journalctl@ceph.osd.3.smithi089.stdout:Jul 15 07:11:46 smithi089 bash[119335]: Error response from daemon: No such container: ceph-75ba02bc-4274-11ef-bcad-c7b262605968-osd-3 2024-07-15T07:11:46.558 INFO:journalctl@ceph.osd.4.smithi134.stdout:Jul 15 07:11:46 smithi134 bash[73271]: debug 2024-07-15T07:11:46.217+0000 7fcf66132640 -1 osd.4 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:18.715966+0000 front 2024-07-15T07:11:18.715810+0000 (oldest deadline 2024-07-15T07:11:42.116030+0000) 2024-07-15T07:11:47.270 INFO:journalctl@ceph.osd.5.smithi134.stdout:Jul 15 07:11:46 smithi134 bash[33230]: debug 2024-07-15T07:11:46.941+0000 7f19b4257640 -1 osd.5 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:16.769056+0000 front 2024-07-15T07:11:16.769321+0000 (oldest deadline 2024-07-15T07:11:40.269036+0000) 2024-07-15T07:11:47.271 INFO:journalctl@ceph.osd.6.smithi134.stdout:Jul 15 07:11:46 smithi134 bash[38841]: debug 2024-07-15T07:11:46.893+0000 7fb73c9d7640 -1 osd.6 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:15.147034+0000 front 2024-07-15T07:11:15.147232+0000 (oldest deadline 2024-07-15T07:11:41.046849+0000) 2024-07-15T07:11:47.271 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:11:46 smithi134 bash[44248]: debug 2024-07-15T07:11:46.909+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:19.694655+0000 front 2024-07-15T07:11:19.695037+0000 (oldest deadline 2024-07-15T07:11:43.794595+0000) 2024-07-15T07:11:47.557 INFO:journalctl@ceph.osd.4.smithi134.stdout:Jul 15 07:11:47 smithi134 bash[73271]: debug 2024-07-15T07:11:47.265+0000 7fcf66132640 -1 osd.4 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:18.715966+0000 front 2024-07-15T07:11:18.715810+0000 (oldest deadline 2024-07-15T07:11:42.116030+0000) 2024-07-15T07:11:48.090 DEBUG:teuthology.orchestra.run.smithi089:> sudo pkill -f 'journalctl -f -n 0 -u ceph-75ba02bc-4274-11ef-bcad-c7b262605968@osd.3.service' 2024-07-15T07:11:48.117 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-07-15T07:11:48.117 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-07-15T07:11:48.117 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-07-15T07:11:48.118 DEBUG:teuthology.orchestra.run.smithi134:> sudo systemctl stop ceph-75ba02bc-4274-11ef-bcad-c7b262605968@osd.4 2024-07-15T07:11:48.128 INFO:journalctl@ceph.osd.6.smithi134.stdout:Jul 15 07:11:47 smithi134 bash[38841]: debug 2024-07-15T07:11:47.861+0000 7fb73c9d7640 -1 osd.6 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:15.147034+0000 front 2024-07-15T07:11:15.147232+0000 (oldest deadline 2024-07-15T07:11:41.046849+0000) 2024-07-15T07:11:48.129 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:11:47 smithi134 bash[44248]: debug 2024-07-15T07:11:47.929+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:19.694655+0000 front 2024-07-15T07:11:19.695037+0000 (oldest deadline 2024-07-15T07:11:43.794595+0000) 2024-07-15T07:11:48.129 INFO:journalctl@ceph.osd.5.smithi134.stdout:Jul 15 07:11:47 smithi134 bash[33230]: debug 2024-07-15T07:11:47.965+0000 7f19b4257640 -1 osd.5 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:16.769056+0000 front 2024-07-15T07:11:16.769321+0000 (oldest deadline 2024-07-15T07:11:40.269036+0000) 2024-07-15T07:11:48.558 INFO:journalctl@ceph.osd.4.smithi134.stdout:Jul 15 07:11:48 smithi134 systemd[1]: Stopping Ceph osd.4 for 75ba02bc-4274-11ef-bcad-c7b262605968... 2024-07-15T07:11:48.558 INFO:journalctl@ceph.osd.4.smithi134.stdout:Jul 15 07:11:48 smithi134 bash[73271]: debug 2024-07-15T07:11:48.245+0000 7fcf66132640 -1 osd.4 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:18.715966+0000 front 2024-07-15T07:11:18.715810+0000 (oldest deadline 2024-07-15T07:11:42.116030+0000) 2024-07-15T07:11:48.561 INFO:journalctl@ceph.osd.4.smithi134.stdout:Jul 15 07:11:48 smithi134 bash[73271]: debug 2024-07-15T07:11:48.341+0000 7fcf69b1e640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-07-15T07:11:48.561 INFO:journalctl@ceph.osd.4.smithi134.stdout:Jul 15 07:11:48 smithi134 bash[73271]: debug 2024-07-15T07:11:48.341+0000 7fcf69b1e640 -1 osd.4 114 *** Got signal Terminated *** 2024-07-15T07:11:48.562 INFO:journalctl@ceph.osd.4.smithi134.stdout:Jul 15 07:11:48 smithi134 bash[73271]: debug 2024-07-15T07:11:48.341+0000 7fcf69b1e640 -1 osd.4 114 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-07-15T07:11:49.248 INFO:journalctl@ceph.osd.6.smithi134.stdout:Jul 15 07:11:48 smithi134 bash[38841]: debug 2024-07-15T07:11:48.869+0000 7fb73c9d7640 -1 osd.6 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:15.147034+0000 front 2024-07-15T07:11:15.147232+0000 (oldest deadline 2024-07-15T07:11:41.046849+0000) 2024-07-15T07:11:49.249 INFO:journalctl@ceph.osd.5.smithi134.stdout:Jul 15 07:11:48 smithi134 bash[33230]: debug 2024-07-15T07:11:48.953+0000 7f19b4257640 -1 osd.5 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:16.769056+0000 front 2024-07-15T07:11:16.769321+0000 (oldest deadline 2024-07-15T07:11:40.269036+0000) 2024-07-15T07:11:49.249 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:11:48 smithi134 bash[44248]: debug 2024-07-15T07:11:48.889+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:19.694655+0000 front 2024-07-15T07:11:19.695037+0000 (oldest deadline 2024-07-15T07:11:43.794595+0000) 2024-07-15T07:11:49.557 INFO:journalctl@ceph.osd.4.smithi134.stdout:Jul 15 07:11:49 smithi134 bash[73271]: debug 2024-07-15T07:11:49.249+0000 7fcf66132640 -1 osd.4 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:18.715966+0000 front 2024-07-15T07:11:18.715810+0000 (oldest deadline 2024-07-15T07:11:42.116030+0000) 2024-07-15T07:11:50.217 INFO:journalctl@ceph.osd.6.smithi134.stdout:Jul 15 07:11:49 smithi134 bash[38841]: debug 2024-07-15T07:11:49.873+0000 7fb73c9d7640 -1 osd.6 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:15.147034+0000 front 2024-07-15T07:11:15.147232+0000 (oldest deadline 2024-07-15T07:11:41.046849+0000) 2024-07-15T07:11:50.217 INFO:journalctl@ceph.osd.6.smithi134.stdout:Jul 15 07:11:49 smithi134 bash[38841]: debug 2024-07-15T07:11:49.873+0000 7fb73c9d7640 -1 osd.6 114 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-07-15T07:11:26.849086+0000 front 2024-07-15T07:11:26.849074+0000 (oldest deadline 2024-07-15T07:11:49.049122+0000) 2024-07-15T07:11:50.218 INFO:journalctl@ceph.osd.5.smithi134.stdout:Jul 15 07:11:49 smithi134 bash[33230]: debug 2024-07-15T07:11:49.973+0000 7f19b4257640 -1 osd.5 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:16.769056+0000 front 2024-07-15T07:11:16.769321+0000 (oldest deadline 2024-07-15T07:11:40.269036+0000) 2024-07-15T07:11:50.218 INFO:journalctl@ceph.osd.5.smithi134.stdout:Jul 15 07:11:49 smithi134 bash[33230]: debug 2024-07-15T07:11:49.973+0000 7f19b4257640 -1 osd.5 114 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-07-15T07:11:26.671174+0000 front 2024-07-15T07:11:26.671095+0000 (oldest deadline 2024-07-15T07:11:48.970816+0000) 2024-07-15T07:11:50.218 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:11:49 smithi134 bash[44248]: debug 2024-07-15T07:11:49.917+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:19.694655+0000 front 2024-07-15T07:11:19.695037+0000 (oldest deadline 2024-07-15T07:11:43.794595+0000) 2024-07-15T07:11:50.218 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:11:49 smithi134 bash[44248]: debug 2024-07-15T07:11:49.917+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-07-15T07:11:27.896052+0000 front 2024-07-15T07:11:27.896379+0000 (oldest deadline 2024-07-15T07:11:49.595985+0000) 2024-07-15T07:11:50.558 INFO:journalctl@ceph.osd.4.smithi134.stdout:Jul 15 07:11:50 smithi134 bash[73271]: debug 2024-07-15T07:11:50.217+0000 7fcf66132640 -1 osd.4 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:18.715966+0000 front 2024-07-15T07:11:18.715810+0000 (oldest deadline 2024-07-15T07:11:42.116030+0000) 2024-07-15T07:11:50.621 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 07:11:50 smithi089 bash[72306]: ts=2024-07-15T07:11:50.176Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi089.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.89:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi134.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.134:8443: connect: connection refused" 2024-07-15T07:11:50.622 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 07:11:50 smithi089 bash[72306]: ts=2024-07-15T07:11:50.176Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi089.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.89:8443: connect: connection refused" 2024-07-15T07:11:50.622 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 07:11:50 smithi089 bash[72306]: ts=2024-07-15T07:11:50.178Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi134.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.134:8443: connect: connection refused" 2024-07-15T07:11:51.219 INFO:journalctl@ceph.osd.6.smithi134.stdout:Jul 15 07:11:50 smithi134 bash[38841]: debug 2024-07-15T07:11:50.869+0000 7fb73c9d7640 -1 osd.6 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:15.147034+0000 front 2024-07-15T07:11:15.147232+0000 (oldest deadline 2024-07-15T07:11:41.046849+0000) 2024-07-15T07:11:51.219 INFO:journalctl@ceph.osd.6.smithi134.stdout:Jul 15 07:11:50 smithi134 bash[38841]: debug 2024-07-15T07:11:50.869+0000 7fb73c9d7640 -1 osd.6 114 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-07-15T07:11:26.849086+0000 front 2024-07-15T07:11:26.849074+0000 (oldest deadline 2024-07-15T07:11:49.049122+0000) 2024-07-15T07:11:51.219 INFO:journalctl@ceph.osd.5.smithi134.stdout:Jul 15 07:11:51 smithi134 bash[33230]: debug 2024-07-15T07:11:50.973+0000 7f19b4257640 -1 osd.5 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:16.769056+0000 front 2024-07-15T07:11:16.769321+0000 (oldest deadline 2024-07-15T07:11:40.269036+0000) 2024-07-15T07:11:51.219 INFO:journalctl@ceph.osd.5.smithi134.stdout:Jul 15 07:11:51 smithi134 bash[33230]: debug 2024-07-15T07:11:50.973+0000 7f19b4257640 -1 osd.5 114 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-07-15T07:11:26.671174+0000 front 2024-07-15T07:11:26.671095+0000 (oldest deadline 2024-07-15T07:11:48.970816+0000) 2024-07-15T07:11:51.220 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:11:51 smithi134 bash[44248]: debug 2024-07-15T07:11:50.897+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:19.694655+0000 front 2024-07-15T07:11:19.695037+0000 (oldest deadline 2024-07-15T07:11:43.794595+0000) 2024-07-15T07:11:51.220 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:11:51 smithi134 bash[44248]: debug 2024-07-15T07:11:50.897+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-07-15T07:11:27.896052+0000 front 2024-07-15T07:11:27.896379+0000 (oldest deadline 2024-07-15T07:11:49.595985+0000) 2024-07-15T07:11:51.558 INFO:journalctl@ceph.osd.4.smithi134.stdout:Jul 15 07:11:51 smithi134 bash[73271]: debug 2024-07-15T07:11:51.217+0000 7fcf66132640 -1 osd.4 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:18.715966+0000 front 2024-07-15T07:11:18.715810+0000 (oldest deadline 2024-07-15T07:11:42.116030+0000) 2024-07-15T07:11:51.558 INFO:journalctl@ceph.osd.4.smithi134.stdout:Jul 15 07:11:51 smithi134 bash[73271]: debug 2024-07-15T07:11:51.217+0000 7fcf66132640 -1 osd.4 114 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-07-15T07:11:26.817133+0000 front 2024-07-15T07:11:26.817474+0000 (oldest deadline 2024-07-15T07:11:50.917086+0000) 2024-07-15T07:11:52.212 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:11:51 smithi134 bash[44248]: debug 2024-07-15T07:11:51.913+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:19.694655+0000 front 2024-07-15T07:11:19.695037+0000 (oldest deadline 2024-07-15T07:11:43.794595+0000) 2024-07-15T07:11:52.212 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:11:51 smithi134 bash[44248]: debug 2024-07-15T07:11:51.913+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-07-15T07:11:27.896052+0000 front 2024-07-15T07:11:27.896379+0000 (oldest deadline 2024-07-15T07:11:49.595985+0000) 2024-07-15T07:11:52.212 INFO:journalctl@ceph.osd.5.smithi134.stdout:Jul 15 07:11:52 smithi134 bash[33230]: debug 2024-07-15T07:11:52.017+0000 7f19b4257640 -1 osd.5 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:16.769056+0000 front 2024-07-15T07:11:16.769321+0000 (oldest deadline 2024-07-15T07:11:40.269036+0000) 2024-07-15T07:11:52.213 INFO:journalctl@ceph.osd.5.smithi134.stdout:Jul 15 07:11:52 smithi134 bash[33230]: debug 2024-07-15T07:11:52.017+0000 7f19b4257640 -1 osd.5 114 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-07-15T07:11:26.671174+0000 front 2024-07-15T07:11:26.671095+0000 (oldest deadline 2024-07-15T07:11:48.970816+0000) 2024-07-15T07:11:52.213 INFO:journalctl@ceph.osd.6.smithi134.stdout:Jul 15 07:11:51 smithi134 bash[38841]: debug 2024-07-15T07:11:51.885+0000 7fb73c9d7640 -1 osd.6 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:15.147034+0000 front 2024-07-15T07:11:15.147232+0000 (oldest deadline 2024-07-15T07:11:41.046849+0000) 2024-07-15T07:11:52.213 INFO:journalctl@ceph.osd.6.smithi134.stdout:Jul 15 07:11:51 smithi134 bash[38841]: debug 2024-07-15T07:11:51.885+0000 7fb73c9d7640 -1 osd.6 114 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-07-15T07:11:26.849086+0000 front 2024-07-15T07:11:26.849074+0000 (oldest deadline 2024-07-15T07:11:49.049122+0000) 2024-07-15T07:11:52.558 INFO:journalctl@ceph.osd.4.smithi134.stdout:Jul 15 07:11:52 smithi134 bash[73271]: debug 2024-07-15T07:11:52.213+0000 7fcf66132640 -1 osd.4 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:18.715966+0000 front 2024-07-15T07:11:18.715810+0000 (oldest deadline 2024-07-15T07:11:42.116030+0000) 2024-07-15T07:11:52.558 INFO:journalctl@ceph.osd.4.smithi134.stdout:Jul 15 07:11:52 smithi134 bash[73271]: debug 2024-07-15T07:11:52.213+0000 7fcf66132640 -1 osd.4 114 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-07-15T07:11:26.817133+0000 front 2024-07-15T07:11:26.817474+0000 (oldest deadline 2024-07-15T07:11:50.917086+0000) 2024-07-15T07:11:53.202 INFO:journalctl@ceph.osd.6.smithi134.stdout:Jul 15 07:11:52 smithi134 bash[38841]: debug 2024-07-15T07:11:52.913+0000 7fb73c9d7640 -1 osd.6 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:15.147034+0000 front 2024-07-15T07:11:15.147232+0000 (oldest deadline 2024-07-15T07:11:41.046849+0000) 2024-07-15T07:11:53.202 INFO:journalctl@ceph.osd.6.smithi134.stdout:Jul 15 07:11:52 smithi134 bash[38841]: debug 2024-07-15T07:11:52.913+0000 7fb73c9d7640 -1 osd.6 114 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-07-15T07:11:26.849086+0000 front 2024-07-15T07:11:26.849074+0000 (oldest deadline 2024-07-15T07:11:49.049122+0000) 2024-07-15T07:11:53.203 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:11:52 smithi134 bash[44248]: debug 2024-07-15T07:11:52.885+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:19.694655+0000 front 2024-07-15T07:11:19.695037+0000 (oldest deadline 2024-07-15T07:11:43.794595+0000) 2024-07-15T07:11:53.203 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:11:52 smithi134 bash[44248]: debug 2024-07-15T07:11:52.885+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-07-15T07:11:27.896052+0000 front 2024-07-15T07:11:27.896379+0000 (oldest deadline 2024-07-15T07:11:49.595985+0000) 2024-07-15T07:11:53.203 INFO:journalctl@ceph.osd.5.smithi134.stdout:Jul 15 07:11:52 smithi134 bash[33230]: debug 2024-07-15T07:11:52.985+0000 7f19b4257640 -1 osd.5 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:16.769056+0000 front 2024-07-15T07:11:16.769321+0000 (oldest deadline 2024-07-15T07:11:40.269036+0000) 2024-07-15T07:11:53.203 INFO:journalctl@ceph.osd.5.smithi134.stdout:Jul 15 07:11:52 smithi134 bash[33230]: debug 2024-07-15T07:11:52.985+0000 7f19b4257640 -1 osd.5 114 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-07-15T07:11:26.671174+0000 front 2024-07-15T07:11:26.671095+0000 (oldest deadline 2024-07-15T07:11:48.970816+0000) 2024-07-15T07:11:53.557 INFO:journalctl@ceph.osd.4.smithi134.stdout:Jul 15 07:11:53 smithi134 bash[73271]: debug 2024-07-15T07:11:53.201+0000 7fcf66132640 -1 osd.4 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:18.715966+0000 front 2024-07-15T07:11:18.715810+0000 (oldest deadline 2024-07-15T07:11:42.116030+0000) 2024-07-15T07:11:53.558 INFO:journalctl@ceph.osd.4.smithi134.stdout:Jul 15 07:11:53 smithi134 bash[73271]: debug 2024-07-15T07:11:53.201+0000 7fcf66132640 -1 osd.4 114 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-07-15T07:11:26.817133+0000 front 2024-07-15T07:11:26.817474+0000 (oldest deadline 2024-07-15T07:11:50.917086+0000) 2024-07-15T07:11:54.058 INFO:journalctl@ceph.osd.5.smithi134.stdout:Jul 15 07:11:54 smithi134 bash[33230]: debug 2024-07-15T07:11:53.993+0000 7f19b4257640 -1 osd.5 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:16.769056+0000 front 2024-07-15T07:11:16.769321+0000 (oldest deadline 2024-07-15T07:11:40.269036+0000) 2024-07-15T07:11:54.058 INFO:journalctl@ceph.osd.5.smithi134.stdout:Jul 15 07:11:54 smithi134 bash[33230]: debug 2024-07-15T07:11:53.993+0000 7f19b4257640 -1 osd.5 114 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-07-15T07:11:26.671174+0000 front 2024-07-15T07:11:26.671095+0000 (oldest deadline 2024-07-15T07:11:48.970816+0000) 2024-07-15T07:11:54.058 INFO:journalctl@ceph.osd.6.smithi134.stdout:Jul 15 07:11:53 smithi134 bash[38841]: debug 2024-07-15T07:11:53.929+0000 7fb73c9d7640 -1 osd.6 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:15.147034+0000 front 2024-07-15T07:11:15.147232+0000 (oldest deadline 2024-07-15T07:11:41.046849+0000) 2024-07-15T07:11:54.058 INFO:journalctl@ceph.osd.6.smithi134.stdout:Jul 15 07:11:53 smithi134 bash[38841]: debug 2024-07-15T07:11:53.929+0000 7fb73c9d7640 -1 osd.6 114 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-07-15T07:11:26.849086+0000 front 2024-07-15T07:11:26.849074+0000 (oldest deadline 2024-07-15T07:11:49.049122+0000) 2024-07-15T07:11:54.059 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:11:53 smithi134 bash[44248]: debug 2024-07-15T07:11:53.881+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:19.694655+0000 front 2024-07-15T07:11:19.695037+0000 (oldest deadline 2024-07-15T07:11:43.794595+0000) 2024-07-15T07:11:54.059 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:11:53 smithi134 bash[44248]: debug 2024-07-15T07:11:53.881+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-07-15T07:11:27.896052+0000 front 2024-07-15T07:11:27.896379+0000 (oldest deadline 2024-07-15T07:11:49.595985+0000) 2024-07-15T07:11:54.736 INFO:journalctl@ceph.osd.4.smithi134.stdout:Jul 15 07:11:54 smithi134 bash[79432]: ceph-75ba02bc-4274-11ef-bcad-c7b262605968-osd-4 2024-07-15T07:11:54.736 INFO:journalctl@ceph.osd.4.smithi134.stdout:Jul 15 07:11:54 smithi134 bash[79501]: Error response from daemon: No such container: ceph-75ba02bc-4274-11ef-bcad-c7b262605968-osd-4 2024-07-15T07:11:55.058 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:11:54 smithi134 bash[44248]: debug 2024-07-15T07:11:54.853+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:19.694655+0000 front 2024-07-15T07:11:19.695037+0000 (oldest deadline 2024-07-15T07:11:43.794595+0000) 2024-07-15T07:11:55.058 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:11:54 smithi134 bash[44248]: debug 2024-07-15T07:11:54.853+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-07-15T07:11:27.896052+0000 front 2024-07-15T07:11:27.896379+0000 (oldest deadline 2024-07-15T07:11:49.595985+0000) 2024-07-15T07:11:55.058 INFO:journalctl@ceph.osd.5.smithi134.stdout:Jul 15 07:11:54 smithi134 bash[33230]: debug 2024-07-15T07:11:54.969+0000 7f19b4257640 -1 osd.5 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:16.769056+0000 front 2024-07-15T07:11:16.769321+0000 (oldest deadline 2024-07-15T07:11:40.269036+0000) 2024-07-15T07:11:55.058 INFO:journalctl@ceph.osd.5.smithi134.stdout:Jul 15 07:11:54 smithi134 bash[33230]: debug 2024-07-15T07:11:54.969+0000 7f19b4257640 -1 osd.5 114 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-07-15T07:11:26.671174+0000 front 2024-07-15T07:11:26.671095+0000 (oldest deadline 2024-07-15T07:11:48.970816+0000) 2024-07-15T07:11:55.059 INFO:journalctl@ceph.osd.6.smithi134.stdout:Jul 15 07:11:54 smithi134 bash[38841]: debug 2024-07-15T07:11:54.933+0000 7fb73c9d7640 -1 osd.6 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:15.147034+0000 front 2024-07-15T07:11:15.147232+0000 (oldest deadline 2024-07-15T07:11:41.046849+0000) 2024-07-15T07:11:55.059 INFO:journalctl@ceph.osd.6.smithi134.stdout:Jul 15 07:11:54 smithi134 bash[38841]: debug 2024-07-15T07:11:54.933+0000 7fb73c9d7640 -1 osd.6 114 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-07-15T07:11:26.849086+0000 front 2024-07-15T07:11:26.849074+0000 (oldest deadline 2024-07-15T07:11:49.049122+0000) 2024-07-15T07:11:56.111 INFO:journalctl@ceph.osd.5.smithi134.stdout:Jul 15 07:11:56 smithi134 bash[33230]: debug 2024-07-15T07:11:56.005+0000 7f19b4257640 -1 osd.5 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:16.769056+0000 front 2024-07-15T07:11:16.769321+0000 (oldest deadline 2024-07-15T07:11:40.269036+0000) 2024-07-15T07:11:56.112 INFO:journalctl@ceph.osd.5.smithi134.stdout:Jul 15 07:11:56 smithi134 bash[33230]: debug 2024-07-15T07:11:56.005+0000 7f19b4257640 -1 osd.5 114 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-07-15T07:11:26.671174+0000 front 2024-07-15T07:11:26.671095+0000 (oldest deadline 2024-07-15T07:11:48.970816+0000) 2024-07-15T07:11:56.112 INFO:journalctl@ceph.osd.6.smithi134.stdout:Jul 15 07:11:55 smithi134 bash[38841]: debug 2024-07-15T07:11:55.933+0000 7fb73c9d7640 -1 osd.6 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:15.147034+0000 front 2024-07-15T07:11:15.147232+0000 (oldest deadline 2024-07-15T07:11:41.046849+0000) 2024-07-15T07:11:56.112 INFO:journalctl@ceph.osd.6.smithi134.stdout:Jul 15 07:11:55 smithi134 bash[38841]: debug 2024-07-15T07:11:55.933+0000 7fb73c9d7640 -1 osd.6 114 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-07-15T07:11:26.849086+0000 front 2024-07-15T07:11:26.849074+0000 (oldest deadline 2024-07-15T07:11:49.049122+0000) 2024-07-15T07:11:56.112 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:11:55 smithi134 bash[44248]: debug 2024-07-15T07:11:55.873+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:19.694655+0000 front 2024-07-15T07:11:19.695037+0000 (oldest deadline 2024-07-15T07:11:43.794595+0000) 2024-07-15T07:11:56.112 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:11:55 smithi134 bash[44248]: debug 2024-07-15T07:11:55.873+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-07-15T07:11:27.896052+0000 front 2024-07-15T07:11:27.896379+0000 (oldest deadline 2024-07-15T07:11:49.595985+0000) 2024-07-15T07:11:56.806 DEBUG:teuthology.orchestra.run.smithi134:> sudo pkill -f 'journalctl -f -n 0 -u ceph-75ba02bc-4274-11ef-bcad-c7b262605968@osd.4.service' 2024-07-15T07:11:56.874 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-07-15T07:11:56.874 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-07-15T07:11:56.874 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-07-15T07:11:56.874 DEBUG:teuthology.orchestra.run.smithi134:> sudo systemctl stop ceph-75ba02bc-4274-11ef-bcad-c7b262605968@osd.5 2024-07-15T07:11:57.057 INFO:journalctl@ceph.osd.6.smithi134.stdout:Jul 15 07:11:56 smithi134 bash[38841]: debug 2024-07-15T07:11:56.937+0000 7fb73c9d7640 -1 osd.6 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:15.147034+0000 front 2024-07-15T07:11:15.147232+0000 (oldest deadline 2024-07-15T07:11:41.046849+0000) 2024-07-15T07:11:57.058 INFO:journalctl@ceph.osd.6.smithi134.stdout:Jul 15 07:11:56 smithi134 bash[38841]: debug 2024-07-15T07:11:56.937+0000 7fb73c9d7640 -1 osd.6 114 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-07-15T07:11:26.849086+0000 front 2024-07-15T07:11:26.849074+0000 (oldest deadline 2024-07-15T07:11:49.049122+0000) 2024-07-15T07:11:57.058 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:11:56 smithi134 bash[44248]: debug 2024-07-15T07:11:56.857+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:19.694655+0000 front 2024-07-15T07:11:19.695037+0000 (oldest deadline 2024-07-15T07:11:43.794595+0000) 2024-07-15T07:11:57.058 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:11:56 smithi134 bash[44248]: debug 2024-07-15T07:11:56.857+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-07-15T07:11:27.896052+0000 front 2024-07-15T07:11:27.896379+0000 (oldest deadline 2024-07-15T07:11:49.595985+0000) 2024-07-15T07:11:57.059 INFO:journalctl@ceph.osd.5.smithi134.stdout:Jul 15 07:11:56 smithi134 systemd[1]: Stopping Ceph osd.5 for 75ba02bc-4274-11ef-bcad-c7b262605968... 2024-07-15T07:11:57.059 INFO:journalctl@ceph.osd.5.smithi134.stdout:Jul 15 07:11:57 smithi134 bash[33230]: debug 2024-07-15T07:11:57.041+0000 7f19b4257640 -1 osd.5 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:16.769056+0000 front 2024-07-15T07:11:16.769321+0000 (oldest deadline 2024-07-15T07:11:40.269036+0000) 2024-07-15T07:11:57.059 INFO:journalctl@ceph.osd.5.smithi134.stdout:Jul 15 07:11:57 smithi134 bash[33230]: debug 2024-07-15T07:11:57.041+0000 7f19b4257640 -1 osd.5 114 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-07-15T07:11:26.671174+0000 front 2024-07-15T07:11:26.671095+0000 (oldest deadline 2024-07-15T07:11:48.970816+0000) 2024-07-15T07:11:57.059 INFO:journalctl@ceph.osd.5.smithi134.stdout:Jul 15 07:11:57 smithi134 bash[33230]: debug 2024-07-15T07:11:57.045+0000 7f19b8c41640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-07-15T07:11:57.059 INFO:journalctl@ceph.osd.5.smithi134.stdout:Jul 15 07:11:57 smithi134 bash[33230]: debug 2024-07-15T07:11:57.045+0000 7f19b8c41640 -1 osd.5 114 *** Got signal Terminated *** 2024-07-15T07:11:57.059 INFO:journalctl@ceph.osd.5.smithi134.stdout:Jul 15 07:11:57 smithi134 bash[33230]: debug 2024-07-15T07:11:57.045+0000 7f19b8c41640 -1 osd.5 114 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-07-15T07:11:58.308 INFO:journalctl@ceph.osd.6.smithi134.stdout:Jul 15 07:11:57 smithi134 bash[38841]: debug 2024-07-15T07:11:57.969+0000 7fb73c9d7640 -1 osd.6 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:15.147034+0000 front 2024-07-15T07:11:15.147232+0000 (oldest deadline 2024-07-15T07:11:41.046849+0000) 2024-07-15T07:11:58.308 INFO:journalctl@ceph.osd.6.smithi134.stdout:Jul 15 07:11:57 smithi134 bash[38841]: debug 2024-07-15T07:11:57.969+0000 7fb73c9d7640 -1 osd.6 114 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-07-15T07:11:26.849086+0000 front 2024-07-15T07:11:26.849074+0000 (oldest deadline 2024-07-15T07:11:49.049122+0000) 2024-07-15T07:11:58.308 INFO:journalctl@ceph.osd.6.smithi134.stdout:Jul 15 07:11:57 smithi134 bash[38841]: debug 2024-07-15T07:11:57.969+0000 7fb73c9d7640 -1 osd.6 114 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-07-15T07:11:32.550488+0000 front 2024-07-15T07:11:32.550583+0000 (oldest deadline 2024-07-15T07:11:57.250329+0000) 2024-07-15T07:11:58.308 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:11:57 smithi134 bash[44248]: debug 2024-07-15T07:11:57.809+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:19.694655+0000 front 2024-07-15T07:11:19.695037+0000 (oldest deadline 2024-07-15T07:11:43.794595+0000) 2024-07-15T07:11:58.309 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:11:57 smithi134 bash[44248]: debug 2024-07-15T07:11:57.809+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-07-15T07:11:27.896052+0000 front 2024-07-15T07:11:27.896379+0000 (oldest deadline 2024-07-15T07:11:49.595985+0000) 2024-07-15T07:11:58.309 INFO:journalctl@ceph.osd.5.smithi134.stdout:Jul 15 07:11:58 smithi134 bash[33230]: debug 2024-07-15T07:11:58.025+0000 7f19b4257640 -1 osd.5 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:16.769056+0000 front 2024-07-15T07:11:16.769321+0000 (oldest deadline 2024-07-15T07:11:40.269036+0000) 2024-07-15T07:11:58.309 INFO:journalctl@ceph.osd.5.smithi134.stdout:Jul 15 07:11:58 smithi134 bash[33230]: debug 2024-07-15T07:11:58.025+0000 7f19b4257640 -1 osd.5 114 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-07-15T07:11:26.671174+0000 front 2024-07-15T07:11:26.671095+0000 (oldest deadline 2024-07-15T07:11:48.970816+0000) 2024-07-15T07:11:59.308 INFO:journalctl@ceph.osd.6.smithi134.stdout:Jul 15 07:11:58 smithi134 bash[38841]: debug 2024-07-15T07:11:58.961+0000 7fb73c9d7640 -1 osd.6 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:15.147034+0000 front 2024-07-15T07:11:15.147232+0000 (oldest deadline 2024-07-15T07:11:41.046849+0000) 2024-07-15T07:11:59.308 INFO:journalctl@ceph.osd.6.smithi134.stdout:Jul 15 07:11:58 smithi134 bash[38841]: debug 2024-07-15T07:11:58.961+0000 7fb73c9d7640 -1 osd.6 114 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-07-15T07:11:26.849086+0000 front 2024-07-15T07:11:26.849074+0000 (oldest deadline 2024-07-15T07:11:49.049122+0000) 2024-07-15T07:11:59.308 INFO:journalctl@ceph.osd.6.smithi134.stdout:Jul 15 07:11:58 smithi134 bash[38841]: debug 2024-07-15T07:11:58.961+0000 7fb73c9d7640 -1 osd.6 114 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-07-15T07:11:32.550488+0000 front 2024-07-15T07:11:32.550583+0000 (oldest deadline 2024-07-15T07:11:57.250329+0000) 2024-07-15T07:11:59.308 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:11:58 smithi134 bash[44248]: debug 2024-07-15T07:11:58.817+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:19.694655+0000 front 2024-07-15T07:11:19.695037+0000 (oldest deadline 2024-07-15T07:11:43.794595+0000) 2024-07-15T07:11:59.309 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:11:58 smithi134 bash[44248]: debug 2024-07-15T07:11:58.817+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-07-15T07:11:27.896052+0000 front 2024-07-15T07:11:27.896379+0000 (oldest deadline 2024-07-15T07:11:49.595985+0000) 2024-07-15T07:11:59.309 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:11:58 smithi134 bash[44248]: debug 2024-07-15T07:11:58.817+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-07-15T07:11:34.198177+0000 front 2024-07-15T07:11:34.198069+0000 (oldest deadline 2024-07-15T07:11:58.297764+0000) 2024-07-15T07:11:59.309 INFO:journalctl@ceph.osd.5.smithi134.stdout:Jul 15 07:11:59 smithi134 bash[33230]: debug 2024-07-15T07:11:59.041+0000 7f19b4257640 -1 osd.5 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:16.769056+0000 front 2024-07-15T07:11:16.769321+0000 (oldest deadline 2024-07-15T07:11:40.269036+0000) 2024-07-15T07:11:59.309 INFO:journalctl@ceph.osd.5.smithi134.stdout:Jul 15 07:11:59 smithi134 bash[33230]: debug 2024-07-15T07:11:59.041+0000 7f19b4257640 -1 osd.5 114 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-07-15T07:11:26.671174+0000 front 2024-07-15T07:11:26.671095+0000 (oldest deadline 2024-07-15T07:11:48.970816+0000) 2024-07-15T07:11:59.309 INFO:journalctl@ceph.osd.5.smithi134.stdout:Jul 15 07:11:59 smithi134 bash[33230]: debug 2024-07-15T07:11:59.041+0000 7f19b4257640 -1 osd.5 114 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-07-15T07:11:34.272178+0000 front 2024-07-15T07:11:34.272252+0000 (oldest deadline 2024-07-15T07:11:58.372168+0000) 2024-07-15T07:12:00.058 INFO:journalctl@ceph.osd.5.smithi134.stdout:Jul 15 07:12:00 smithi134 bash[33230]: debug 2024-07-15T07:12:00.013+0000 7f19b4257640 -1 osd.5 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:16.769056+0000 front 2024-07-15T07:11:16.769321+0000 (oldest deadline 2024-07-15T07:11:40.269036+0000) 2024-07-15T07:12:00.058 INFO:journalctl@ceph.osd.5.smithi134.stdout:Jul 15 07:12:00 smithi134 bash[33230]: debug 2024-07-15T07:12:00.013+0000 7f19b4257640 -1 osd.5 114 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-07-15T07:11:26.671174+0000 front 2024-07-15T07:11:26.671095+0000 (oldest deadline 2024-07-15T07:11:48.970816+0000) 2024-07-15T07:12:00.058 INFO:journalctl@ceph.osd.5.smithi134.stdout:Jul 15 07:12:00 smithi134 bash[33230]: debug 2024-07-15T07:12:00.013+0000 7f19b4257640 -1 osd.5 114 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-07-15T07:11:34.272178+0000 front 2024-07-15T07:11:34.272252+0000 (oldest deadline 2024-07-15T07:11:58.372168+0000) 2024-07-15T07:12:00.059 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:11:59 smithi134 bash[44248]: debug 2024-07-15T07:11:59.805+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:19.694655+0000 front 2024-07-15T07:11:19.695037+0000 (oldest deadline 2024-07-15T07:11:43.794595+0000) 2024-07-15T07:12:00.059 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:11:59 smithi134 bash[44248]: debug 2024-07-15T07:11:59.805+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-07-15T07:11:27.896052+0000 front 2024-07-15T07:11:27.896379+0000 (oldest deadline 2024-07-15T07:11:49.595985+0000) 2024-07-15T07:12:00.059 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:11:59 smithi134 bash[44248]: debug 2024-07-15T07:11:59.805+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-07-15T07:11:34.198177+0000 front 2024-07-15T07:11:34.198069+0000 (oldest deadline 2024-07-15T07:11:58.297764+0000) 2024-07-15T07:12:00.059 INFO:journalctl@ceph.osd.6.smithi134.stdout:Jul 15 07:12:00 smithi134 bash[38841]: debug 2024-07-15T07:12:00.001+0000 7fb73c9d7640 -1 osd.6 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:15.147034+0000 front 2024-07-15T07:11:15.147232+0000 (oldest deadline 2024-07-15T07:11:41.046849+0000) 2024-07-15T07:12:00.059 INFO:journalctl@ceph.osd.6.smithi134.stdout:Jul 15 07:12:00 smithi134 bash[38841]: debug 2024-07-15T07:12:00.001+0000 7fb73c9d7640 -1 osd.6 114 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-07-15T07:11:26.849086+0000 front 2024-07-15T07:11:26.849074+0000 (oldest deadline 2024-07-15T07:11:49.049122+0000) 2024-07-15T07:12:00.059 INFO:journalctl@ceph.osd.6.smithi134.stdout:Jul 15 07:12:00 smithi134 bash[38841]: debug 2024-07-15T07:12:00.001+0000 7fb73c9d7640 -1 osd.6 114 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-07-15T07:11:32.550488+0000 front 2024-07-15T07:11:32.550583+0000 (oldest deadline 2024-07-15T07:11:57.250329+0000) 2024-07-15T07:12:00.622 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 07:12:00 smithi089 bash[72306]: ts=2024-07-15T07:12:00.177Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi134.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.134:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi089.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.89:8443: connect: connection refused" 2024-07-15T07:12:00.622 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 07:12:00 smithi089 bash[72306]: ts=2024-07-15T07:12:00.178Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi089.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.89:8443: connect: connection refused" 2024-07-15T07:12:00.622 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 07:12:00 smithi089 bash[72306]: ts=2024-07-15T07:12:00.179Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi134.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.134:8443: connect: connection refused" 2024-07-15T07:12:01.308 INFO:journalctl@ceph.osd.6.smithi134.stdout:Jul 15 07:12:01 smithi134 bash[38841]: debug 2024-07-15T07:12:01.017+0000 7fb73c9d7640 -1 osd.6 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:15.147034+0000 front 2024-07-15T07:11:15.147232+0000 (oldest deadline 2024-07-15T07:11:41.046849+0000) 2024-07-15T07:12:01.308 INFO:journalctl@ceph.osd.6.smithi134.stdout:Jul 15 07:12:01 smithi134 bash[38841]: debug 2024-07-15T07:12:01.017+0000 7fb73c9d7640 -1 osd.6 114 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-07-15T07:11:26.849086+0000 front 2024-07-15T07:11:26.849074+0000 (oldest deadline 2024-07-15T07:11:49.049122+0000) 2024-07-15T07:12:01.308 INFO:journalctl@ceph.osd.6.smithi134.stdout:Jul 15 07:12:01 smithi134 bash[38841]: debug 2024-07-15T07:12:01.017+0000 7fb73c9d7640 -1 osd.6 114 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-07-15T07:11:32.550488+0000 front 2024-07-15T07:11:32.550583+0000 (oldest deadline 2024-07-15T07:11:57.250329+0000) 2024-07-15T07:12:01.309 INFO:journalctl@ceph.osd.5.smithi134.stdout:Jul 15 07:12:01 smithi134 bash[33230]: debug 2024-07-15T07:12:01.037+0000 7f19b4257640 -1 osd.5 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:16.769056+0000 front 2024-07-15T07:11:16.769321+0000 (oldest deadline 2024-07-15T07:11:40.269036+0000) 2024-07-15T07:12:01.309 INFO:journalctl@ceph.osd.5.smithi134.stdout:Jul 15 07:12:01 smithi134 bash[33230]: debug 2024-07-15T07:12:01.037+0000 7f19b4257640 -1 osd.5 114 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-07-15T07:11:26.671174+0000 front 2024-07-15T07:11:26.671095+0000 (oldest deadline 2024-07-15T07:11:48.970816+0000) 2024-07-15T07:12:01.309 INFO:journalctl@ceph.osd.5.smithi134.stdout:Jul 15 07:12:01 smithi134 bash[33230]: debug 2024-07-15T07:12:01.037+0000 7f19b4257640 -1 osd.5 114 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-07-15T07:11:34.272178+0000 front 2024-07-15T07:11:34.272252+0000 (oldest deadline 2024-07-15T07:11:58.372168+0000) 2024-07-15T07:12:01.309 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:12:00 smithi134 bash[44248]: debug 2024-07-15T07:12:00.849+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:19.694655+0000 front 2024-07-15T07:11:19.695037+0000 (oldest deadline 2024-07-15T07:11:43.794595+0000) 2024-07-15T07:12:01.309 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:12:00 smithi134 bash[44248]: debug 2024-07-15T07:12:00.849+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-07-15T07:11:27.896052+0000 front 2024-07-15T07:11:27.896379+0000 (oldest deadline 2024-07-15T07:11:49.595985+0000) 2024-07-15T07:12:01.309 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:12:00 smithi134 bash[44248]: debug 2024-07-15T07:12:00.849+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-07-15T07:11:34.198177+0000 front 2024-07-15T07:11:34.198069+0000 (oldest deadline 2024-07-15T07:11:58.297764+0000) 2024-07-15T07:12:02.308 INFO:journalctl@ceph.osd.6.smithi134.stdout:Jul 15 07:12:01 smithi134 bash[38841]: debug 2024-07-15T07:12:01.969+0000 7fb73c9d7640 -1 osd.6 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:15.147034+0000 front 2024-07-15T07:11:15.147232+0000 (oldest deadline 2024-07-15T07:11:41.046849+0000) 2024-07-15T07:12:02.308 INFO:journalctl@ceph.osd.6.smithi134.stdout:Jul 15 07:12:01 smithi134 bash[38841]: debug 2024-07-15T07:12:01.969+0000 7fb73c9d7640 -1 osd.6 114 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-07-15T07:11:26.849086+0000 front 2024-07-15T07:11:26.849074+0000 (oldest deadline 2024-07-15T07:11:49.049122+0000) 2024-07-15T07:12:02.308 INFO:journalctl@ceph.osd.6.smithi134.stdout:Jul 15 07:12:01 smithi134 bash[38841]: debug 2024-07-15T07:12:01.969+0000 7fb73c9d7640 -1 osd.6 114 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-07-15T07:11:32.550488+0000 front 2024-07-15T07:11:32.550583+0000 (oldest deadline 2024-07-15T07:11:57.250329+0000) 2024-07-15T07:12:02.308 INFO:journalctl@ceph.osd.5.smithi134.stdout:Jul 15 07:12:02 smithi134 bash[33230]: debug 2024-07-15T07:12:02.033+0000 7f19b4257640 -1 osd.5 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:16.769056+0000 front 2024-07-15T07:11:16.769321+0000 (oldest deadline 2024-07-15T07:11:40.269036+0000) 2024-07-15T07:12:02.309 INFO:journalctl@ceph.osd.5.smithi134.stdout:Jul 15 07:12:02 smithi134 bash[33230]: debug 2024-07-15T07:12:02.033+0000 7f19b4257640 -1 osd.5 114 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-07-15T07:11:26.671174+0000 front 2024-07-15T07:11:26.671095+0000 (oldest deadline 2024-07-15T07:11:48.970816+0000) 2024-07-15T07:12:02.309 INFO:journalctl@ceph.osd.5.smithi134.stdout:Jul 15 07:12:02 smithi134 bash[33230]: debug 2024-07-15T07:12:02.033+0000 7f19b4257640 -1 osd.5 114 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-07-15T07:11:34.272178+0000 front 2024-07-15T07:11:34.272252+0000 (oldest deadline 2024-07-15T07:11:58.372168+0000) 2024-07-15T07:12:02.309 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:12:01 smithi134 bash[44248]: debug 2024-07-15T07:12:01.893+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:19.694655+0000 front 2024-07-15T07:11:19.695037+0000 (oldest deadline 2024-07-15T07:11:43.794595+0000) 2024-07-15T07:12:02.309 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:12:01 smithi134 bash[44248]: debug 2024-07-15T07:12:01.893+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-07-15T07:11:27.896052+0000 front 2024-07-15T07:11:27.896379+0000 (oldest deadline 2024-07-15T07:11:49.595985+0000) 2024-07-15T07:12:02.309 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:12:01 smithi134 bash[44248]: debug 2024-07-15T07:12:01.893+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-07-15T07:11:34.198177+0000 front 2024-07-15T07:11:34.198069+0000 (oldest deadline 2024-07-15T07:11:58.297764+0000) 2024-07-15T07:12:03.058 INFO:journalctl@ceph.osd.6.smithi134.stdout:Jul 15 07:12:02 smithi134 bash[38841]: debug 2024-07-15T07:12:02.977+0000 7fb73c9d7640 -1 osd.6 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:15.147034+0000 front 2024-07-15T07:11:15.147232+0000 (oldest deadline 2024-07-15T07:11:41.046849+0000) 2024-07-15T07:12:03.058 INFO:journalctl@ceph.osd.6.smithi134.stdout:Jul 15 07:12:02 smithi134 bash[38841]: debug 2024-07-15T07:12:02.977+0000 7fb73c9d7640 -1 osd.6 114 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-07-15T07:11:26.849086+0000 front 2024-07-15T07:11:26.849074+0000 (oldest deadline 2024-07-15T07:11:49.049122+0000) 2024-07-15T07:12:03.058 INFO:journalctl@ceph.osd.6.smithi134.stdout:Jul 15 07:12:02 smithi134 bash[38841]: debug 2024-07-15T07:12:02.977+0000 7fb73c9d7640 -1 osd.6 114 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-07-15T07:11:32.550488+0000 front 2024-07-15T07:11:32.550583+0000 (oldest deadline 2024-07-15T07:11:57.250329+0000) 2024-07-15T07:12:03.058 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:12:02 smithi134 bash[44248]: debug 2024-07-15T07:12:02.917+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:19.694655+0000 front 2024-07-15T07:11:19.695037+0000 (oldest deadline 2024-07-15T07:11:43.794595+0000) 2024-07-15T07:12:03.059 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:12:02 smithi134 bash[44248]: debug 2024-07-15T07:12:02.917+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-07-15T07:11:27.896052+0000 front 2024-07-15T07:11:27.896379+0000 (oldest deadline 2024-07-15T07:11:49.595985+0000) 2024-07-15T07:12:03.059 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:12:02 smithi134 bash[44248]: debug 2024-07-15T07:12:02.917+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-07-15T07:11:34.198177+0000 front 2024-07-15T07:11:34.198069+0000 (oldest deadline 2024-07-15T07:11:58.297764+0000) 2024-07-15T07:12:03.429 INFO:journalctl@ceph.osd.5.smithi134.stdout:Jul 15 07:12:03 smithi134 bash[79631]: ceph-75ba02bc-4274-11ef-bcad-c7b262605968-osd-5 2024-07-15T07:12:03.430 INFO:journalctl@ceph.osd.5.smithi134.stdout:Jul 15 07:12:03 smithi134 bash[79693]: Error response from daemon: No such container: ceph-75ba02bc-4274-11ef-bcad-c7b262605968-osd-5 2024-07-15T07:12:04.308 INFO:journalctl@ceph.osd.6.smithi134.stdout:Jul 15 07:12:03 smithi134 bash[38841]: debug 2024-07-15T07:12:03.929+0000 7fb73c9d7640 -1 osd.6 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:15.147034+0000 front 2024-07-15T07:11:15.147232+0000 (oldest deadline 2024-07-15T07:11:41.046849+0000) 2024-07-15T07:12:04.308 INFO:journalctl@ceph.osd.6.smithi134.stdout:Jul 15 07:12:03 smithi134 bash[38841]: debug 2024-07-15T07:12:03.929+0000 7fb73c9d7640 -1 osd.6 114 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-07-15T07:11:26.849086+0000 front 2024-07-15T07:11:26.849074+0000 (oldest deadline 2024-07-15T07:11:49.049122+0000) 2024-07-15T07:12:04.308 INFO:journalctl@ceph.osd.6.smithi134.stdout:Jul 15 07:12:03 smithi134 bash[38841]: debug 2024-07-15T07:12:03.929+0000 7fb73c9d7640 -1 osd.6 114 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-07-15T07:11:32.550488+0000 front 2024-07-15T07:11:32.550583+0000 (oldest deadline 2024-07-15T07:11:57.250329+0000) 2024-07-15T07:12:04.309 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:12:03 smithi134 bash[44248]: debug 2024-07-15T07:12:03.961+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:19.694655+0000 front 2024-07-15T07:11:19.695037+0000 (oldest deadline 2024-07-15T07:11:43.794595+0000) 2024-07-15T07:12:04.309 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:12:03 smithi134 bash[44248]: debug 2024-07-15T07:12:03.961+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-07-15T07:11:27.896052+0000 front 2024-07-15T07:11:27.896379+0000 (oldest deadline 2024-07-15T07:11:49.595985+0000) 2024-07-15T07:12:04.309 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:12:03 smithi134 bash[44248]: debug 2024-07-15T07:12:03.961+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-07-15T07:11:34.198177+0000 front 2024-07-15T07:11:34.198069+0000 (oldest deadline 2024-07-15T07:11:58.297764+0000) 2024-07-15T07:12:04.936 INFO:journalctl@ceph.osd.6.smithi134.stdout:Jul 15 07:12:04 smithi134 bash[38841]: debug 2024-07-15T07:12:04.897+0000 7fb73c9d7640 -1 osd.6 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:15.147034+0000 front 2024-07-15T07:11:15.147232+0000 (oldest deadline 2024-07-15T07:11:41.046849+0000) 2024-07-15T07:12:04.936 INFO:journalctl@ceph.osd.6.smithi134.stdout:Jul 15 07:12:04 smithi134 bash[38841]: debug 2024-07-15T07:12:04.897+0000 7fb73c9d7640 -1 osd.6 114 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-07-15T07:11:26.849086+0000 front 2024-07-15T07:11:26.849074+0000 (oldest deadline 2024-07-15T07:11:49.049122+0000) 2024-07-15T07:12:04.936 INFO:journalctl@ceph.osd.6.smithi134.stdout:Jul 15 07:12:04 smithi134 bash[38841]: debug 2024-07-15T07:12:04.897+0000 7fb73c9d7640 -1 osd.6 114 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-07-15T07:11:32.550488+0000 front 2024-07-15T07:11:32.550583+0000 (oldest deadline 2024-07-15T07:11:57.250329+0000) 2024-07-15T07:12:05.308 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:12:04 smithi134 bash[44248]: debug 2024-07-15T07:12:04.977+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:19.694655+0000 front 2024-07-15T07:11:19.695037+0000 (oldest deadline 2024-07-15T07:11:43.794595+0000) 2024-07-15T07:12:05.308 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:12:04 smithi134 bash[44248]: debug 2024-07-15T07:12:04.977+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-07-15T07:11:27.896052+0000 front 2024-07-15T07:11:27.896379+0000 (oldest deadline 2024-07-15T07:11:49.595985+0000) 2024-07-15T07:12:05.308 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:12:04 smithi134 bash[44248]: debug 2024-07-15T07:12:04.977+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-07-15T07:11:34.198177+0000 front 2024-07-15T07:11:34.198069+0000 (oldest deadline 2024-07-15T07:11:58.297764+0000) 2024-07-15T07:12:05.400 DEBUG:teuthology.orchestra.run.smithi134:> sudo pkill -f 'journalctl -f -n 0 -u ceph-75ba02bc-4274-11ef-bcad-c7b262605968@osd.5.service' 2024-07-15T07:12:05.429 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-07-15T07:12:05.429 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-07-15T07:12:05.429 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-07-15T07:12:05.429 DEBUG:teuthology.orchestra.run.smithi134:> sudo systemctl stop ceph-75ba02bc-4274-11ef-bcad-c7b262605968@osd.6 2024-07-15T07:12:05.808 INFO:journalctl@ceph.osd.6.smithi134.stdout:Jul 15 07:12:05 smithi134 systemd[1]: Stopping Ceph osd.6 for 75ba02bc-4274-11ef-bcad-c7b262605968... 2024-07-15T07:12:05.808 INFO:journalctl@ceph.osd.6.smithi134.stdout:Jul 15 07:12:05 smithi134 bash[38841]: debug 2024-07-15T07:12:05.613+0000 7fb7403bf640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-07-15T07:12:05.808 INFO:journalctl@ceph.osd.6.smithi134.stdout:Jul 15 07:12:05 smithi134 bash[38841]: debug 2024-07-15T07:12:05.613+0000 7fb7403bf640 -1 osd.6 114 *** Got signal Terminated *** 2024-07-15T07:12:05.808 INFO:journalctl@ceph.osd.6.smithi134.stdout:Jul 15 07:12:05 smithi134 bash[38841]: debug 2024-07-15T07:12:05.613+0000 7fb7403bf640 -1 osd.6 114 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-07-15T07:12:06.308 INFO:journalctl@ceph.osd.6.smithi134.stdout:Jul 15 07:12:05 smithi134 bash[38841]: debug 2024-07-15T07:12:05.921+0000 7fb73c9d7640 -1 osd.6 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:15.147034+0000 front 2024-07-15T07:11:15.147232+0000 (oldest deadline 2024-07-15T07:11:41.046849+0000) 2024-07-15T07:12:06.308 INFO:journalctl@ceph.osd.6.smithi134.stdout:Jul 15 07:12:05 smithi134 bash[38841]: debug 2024-07-15T07:12:05.921+0000 7fb73c9d7640 -1 osd.6 114 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-07-15T07:11:26.849086+0000 front 2024-07-15T07:11:26.849074+0000 (oldest deadline 2024-07-15T07:11:49.049122+0000) 2024-07-15T07:12:06.308 INFO:journalctl@ceph.osd.6.smithi134.stdout:Jul 15 07:12:05 smithi134 bash[38841]: debug 2024-07-15T07:12:05.921+0000 7fb73c9d7640 -1 osd.6 114 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-07-15T07:11:32.550488+0000 front 2024-07-15T07:11:32.550583+0000 (oldest deadline 2024-07-15T07:11:57.250329+0000) 2024-07-15T07:12:06.308 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:12:06 smithi134 bash[44248]: debug 2024-07-15T07:12:05.997+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:19.694655+0000 front 2024-07-15T07:11:19.695037+0000 (oldest deadline 2024-07-15T07:11:43.794595+0000) 2024-07-15T07:12:06.309 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:12:06 smithi134 bash[44248]: debug 2024-07-15T07:12:05.997+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-07-15T07:11:27.896052+0000 front 2024-07-15T07:11:27.896379+0000 (oldest deadline 2024-07-15T07:11:49.595985+0000) 2024-07-15T07:12:06.309 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:12:06 smithi134 bash[44248]: debug 2024-07-15T07:12:05.997+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-07-15T07:11:34.198177+0000 front 2024-07-15T07:11:34.198069+0000 (oldest deadline 2024-07-15T07:11:58.297764+0000) 2024-07-15T07:12:07.308 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:12:07 smithi134 bash[44248]: debug 2024-07-15T07:12:07.009+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:19.694655+0000 front 2024-07-15T07:11:19.695037+0000 (oldest deadline 2024-07-15T07:11:43.794595+0000) 2024-07-15T07:12:07.308 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:12:07 smithi134 bash[44248]: debug 2024-07-15T07:12:07.009+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-07-15T07:11:27.896052+0000 front 2024-07-15T07:11:27.896379+0000 (oldest deadline 2024-07-15T07:11:49.595985+0000) 2024-07-15T07:12:07.308 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:12:07 smithi134 bash[44248]: debug 2024-07-15T07:12:07.009+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-07-15T07:11:34.198177+0000 front 2024-07-15T07:11:34.198069+0000 (oldest deadline 2024-07-15T07:11:58.297764+0000) 2024-07-15T07:12:07.308 INFO:journalctl@ceph.osd.6.smithi134.stdout:Jul 15 07:12:06 smithi134 bash[38841]: debug 2024-07-15T07:12:06.957+0000 7fb73c9d7640 -1 osd.6 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:15.147034+0000 front 2024-07-15T07:11:15.147232+0000 (oldest deadline 2024-07-15T07:11:41.046849+0000) 2024-07-15T07:12:07.309 INFO:journalctl@ceph.osd.6.smithi134.stdout:Jul 15 07:12:06 smithi134 bash[38841]: debug 2024-07-15T07:12:06.957+0000 7fb73c9d7640 -1 osd.6 114 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-07-15T07:11:26.849086+0000 front 2024-07-15T07:11:26.849074+0000 (oldest deadline 2024-07-15T07:11:49.049122+0000) 2024-07-15T07:12:07.309 INFO:journalctl@ceph.osd.6.smithi134.stdout:Jul 15 07:12:06 smithi134 bash[38841]: debug 2024-07-15T07:12:06.957+0000 7fb73c9d7640 -1 osd.6 114 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-07-15T07:11:32.550488+0000 front 2024-07-15T07:11:32.550583+0000 (oldest deadline 2024-07-15T07:11:57.250329+0000) 2024-07-15T07:12:08.308 INFO:journalctl@ceph.osd.6.smithi134.stdout:Jul 15 07:12:08 smithi134 bash[38841]: debug 2024-07-15T07:12:08.005+0000 7fb73c9d7640 -1 osd.6 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:15.147034+0000 front 2024-07-15T07:11:15.147232+0000 (oldest deadline 2024-07-15T07:11:41.046849+0000) 2024-07-15T07:12:08.308 INFO:journalctl@ceph.osd.6.smithi134.stdout:Jul 15 07:12:08 smithi134 bash[38841]: debug 2024-07-15T07:12:08.005+0000 7fb73c9d7640 -1 osd.6 114 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-07-15T07:11:26.849086+0000 front 2024-07-15T07:11:26.849074+0000 (oldest deadline 2024-07-15T07:11:49.049122+0000) 2024-07-15T07:12:08.308 INFO:journalctl@ceph.osd.6.smithi134.stdout:Jul 15 07:12:08 smithi134 bash[38841]: debug 2024-07-15T07:12:08.005+0000 7fb73c9d7640 -1 osd.6 114 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-07-15T07:11:32.550488+0000 front 2024-07-15T07:11:32.550583+0000 (oldest deadline 2024-07-15T07:11:57.250329+0000) 2024-07-15T07:12:08.309 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:12:07 smithi134 bash[44248]: debug 2024-07-15T07:12:07.985+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:19.694655+0000 front 2024-07-15T07:11:19.695037+0000 (oldest deadline 2024-07-15T07:11:43.794595+0000) 2024-07-15T07:12:08.309 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:12:07 smithi134 bash[44248]: debug 2024-07-15T07:12:07.985+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-07-15T07:11:27.896052+0000 front 2024-07-15T07:11:27.896379+0000 (oldest deadline 2024-07-15T07:11:49.595985+0000) 2024-07-15T07:12:08.309 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:12:07 smithi134 bash[44248]: debug 2024-07-15T07:12:07.985+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-07-15T07:11:34.198177+0000 front 2024-07-15T07:11:34.198069+0000 (oldest deadline 2024-07-15T07:11:58.297764+0000) 2024-07-15T07:12:09.308 INFO:journalctl@ceph.osd.6.smithi134.stdout:Jul 15 07:12:08 smithi134 bash[38841]: debug 2024-07-15T07:12:08.993+0000 7fb73c9d7640 -1 osd.6 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:15.147034+0000 front 2024-07-15T07:11:15.147232+0000 (oldest deadline 2024-07-15T07:11:41.046849+0000) 2024-07-15T07:12:09.308 INFO:journalctl@ceph.osd.6.smithi134.stdout:Jul 15 07:12:08 smithi134 bash[38841]: debug 2024-07-15T07:12:08.993+0000 7fb73c9d7640 -1 osd.6 114 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-07-15T07:11:26.849086+0000 front 2024-07-15T07:11:26.849074+0000 (oldest deadline 2024-07-15T07:11:49.049122+0000) 2024-07-15T07:12:09.308 INFO:journalctl@ceph.osd.6.smithi134.stdout:Jul 15 07:12:08 smithi134 bash[38841]: debug 2024-07-15T07:12:08.993+0000 7fb73c9d7640 -1 osd.6 114 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-07-15T07:11:32.550488+0000 front 2024-07-15T07:11:32.550583+0000 (oldest deadline 2024-07-15T07:11:57.250329+0000) 2024-07-15T07:12:09.309 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:12:08 smithi134 bash[44248]: debug 2024-07-15T07:12:08.957+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:19.694655+0000 front 2024-07-15T07:11:19.695037+0000 (oldest deadline 2024-07-15T07:11:43.794595+0000) 2024-07-15T07:12:09.309 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:12:08 smithi134 bash[44248]: debug 2024-07-15T07:12:08.957+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-07-15T07:11:27.896052+0000 front 2024-07-15T07:11:27.896379+0000 (oldest deadline 2024-07-15T07:11:49.595985+0000) 2024-07-15T07:12:09.309 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:12:08 smithi134 bash[44248]: debug 2024-07-15T07:12:08.957+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-07-15T07:11:34.198177+0000 front 2024-07-15T07:11:34.198069+0000 (oldest deadline 2024-07-15T07:11:58.297764+0000) 2024-07-15T07:12:10.308 INFO:journalctl@ceph.osd.6.smithi134.stdout:Jul 15 07:12:09 smithi134 bash[38841]: debug 2024-07-15T07:12:09.968+0000 7fb73c9d7640 -1 osd.6 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:15.147034+0000 front 2024-07-15T07:11:15.147232+0000 (oldest deadline 2024-07-15T07:11:41.046849+0000) 2024-07-15T07:12:10.308 INFO:journalctl@ceph.osd.6.smithi134.stdout:Jul 15 07:12:09 smithi134 bash[38841]: debug 2024-07-15T07:12:09.968+0000 7fb73c9d7640 -1 osd.6 114 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-07-15T07:11:26.849086+0000 front 2024-07-15T07:11:26.849074+0000 (oldest deadline 2024-07-15T07:11:49.049122+0000) 2024-07-15T07:12:10.308 INFO:journalctl@ceph.osd.6.smithi134.stdout:Jul 15 07:12:09 smithi134 bash[38841]: debug 2024-07-15T07:12:09.968+0000 7fb73c9d7640 -1 osd.6 114 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-07-15T07:11:32.550488+0000 front 2024-07-15T07:11:32.550583+0000 (oldest deadline 2024-07-15T07:11:57.250329+0000) 2024-07-15T07:12:10.308 INFO:journalctl@ceph.osd.6.smithi134.stdout:Jul 15 07:12:09 smithi134 bash[38841]: debug 2024-07-15T07:12:09.968+0000 7fb73c9d7640 -1 osd.6 114 heartbeat_check: no reply from 172.21.15.89:6830 osd.3 since back 2024-07-15T07:11:43.652397+0000 front 2024-07-15T07:11:43.652369+0000 (oldest deadline 2024-07-15T07:12:09.552206+0000) 2024-07-15T07:12:10.309 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:12:09 smithi134 bash[44248]: debug 2024-07-15T07:12:09.936+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:19.694655+0000 front 2024-07-15T07:11:19.695037+0000 (oldest deadline 2024-07-15T07:11:43.794595+0000) 2024-07-15T07:12:10.309 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:12:09 smithi134 bash[44248]: debug 2024-07-15T07:12:09.936+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-07-15T07:11:27.896052+0000 front 2024-07-15T07:11:27.896379+0000 (oldest deadline 2024-07-15T07:11:49.595985+0000) 2024-07-15T07:12:10.309 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:12:09 smithi134 bash[44248]: debug 2024-07-15T07:12:09.936+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-07-15T07:11:34.198177+0000 front 2024-07-15T07:11:34.198069+0000 (oldest deadline 2024-07-15T07:11:58.297764+0000) 2024-07-15T07:12:10.621 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 07:12:10 smithi089 bash[72306]: ts=2024-07-15T07:12:10.177Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi134.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.134:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi089.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.89:8443: connect: connection refused" 2024-07-15T07:12:10.622 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 07:12:10 smithi089 bash[72306]: ts=2024-07-15T07:12:10.178Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi089.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.89:8443: connect: connection refused" 2024-07-15T07:12:10.622 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 07:12:10 smithi089 bash[72306]: ts=2024-07-15T07:12:10.179Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi134.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.134:8443: connect: connection refused" 2024-07-15T07:12:11.277 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:12:10 smithi134 bash[44248]: debug 2024-07-15T07:12:10.916+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:19.694655+0000 front 2024-07-15T07:11:19.695037+0000 (oldest deadline 2024-07-15T07:11:43.794595+0000) 2024-07-15T07:12:11.277 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:12:10 smithi134 bash[44248]: debug 2024-07-15T07:12:10.916+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-07-15T07:11:27.896052+0000 front 2024-07-15T07:11:27.896379+0000 (oldest deadline 2024-07-15T07:11:49.595985+0000) 2024-07-15T07:12:11.277 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:12:10 smithi134 bash[44248]: debug 2024-07-15T07:12:10.916+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-07-15T07:11:34.198177+0000 front 2024-07-15T07:11:34.198069+0000 (oldest deadline 2024-07-15T07:11:58.297764+0000) 2024-07-15T07:12:11.277 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:12:10 smithi134 bash[44248]: debug 2024-07-15T07:12:10.916+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6830 osd.3 since back 2024-07-15T07:11:44.099611+0000 front 2024-07-15T07:11:44.099482+0000 (oldest deadline 2024-07-15T07:12:09.999265+0000) 2024-07-15T07:12:11.886 INFO:journalctl@ceph.osd.6.smithi134.stdout:Jul 15 07:12:11 smithi134 bash[79822]: ceph-75ba02bc-4274-11ef-bcad-c7b262605968-osd-6 2024-07-15T07:12:11.886 INFO:journalctl@ceph.osd.6.smithi134.stdout:Jul 15 07:12:11 smithi134 bash[79885]: Error response from daemon: No such container: ceph-75ba02bc-4274-11ef-bcad-c7b262605968-osd-6 2024-07-15T07:12:12.308 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:12:11 smithi134 bash[44248]: debug 2024-07-15T07:12:11.884+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:19.694655+0000 front 2024-07-15T07:11:19.695037+0000 (oldest deadline 2024-07-15T07:11:43.794595+0000) 2024-07-15T07:12:12.308 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:12:11 smithi134 bash[44248]: debug 2024-07-15T07:12:11.884+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-07-15T07:11:27.896052+0000 front 2024-07-15T07:11:27.896379+0000 (oldest deadline 2024-07-15T07:11:49.595985+0000) 2024-07-15T07:12:12.308 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:12:11 smithi134 bash[44248]: debug 2024-07-15T07:12:11.884+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-07-15T07:11:34.198177+0000 front 2024-07-15T07:11:34.198069+0000 (oldest deadline 2024-07-15T07:11:58.297764+0000) 2024-07-15T07:12:12.308 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:12:11 smithi134 bash[44248]: debug 2024-07-15T07:12:11.884+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6830 osd.3 since back 2024-07-15T07:11:44.099611+0000 front 2024-07-15T07:11:44.099482+0000 (oldest deadline 2024-07-15T07:12:09.999265+0000) 2024-07-15T07:12:13.184 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:12:12 smithi134 bash[44248]: debug 2024-07-15T07:12:12.908+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:19.694655+0000 front 2024-07-15T07:11:19.695037+0000 (oldest deadline 2024-07-15T07:11:43.794595+0000) 2024-07-15T07:12:13.184 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:12:12 smithi134 bash[44248]: debug 2024-07-15T07:12:12.908+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-07-15T07:11:27.896052+0000 front 2024-07-15T07:11:27.896379+0000 (oldest deadline 2024-07-15T07:11:49.595985+0000) 2024-07-15T07:12:13.184 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:12:12 smithi134 bash[44248]: debug 2024-07-15T07:12:12.908+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-07-15T07:11:34.198177+0000 front 2024-07-15T07:11:34.198069+0000 (oldest deadline 2024-07-15T07:11:58.297764+0000) 2024-07-15T07:12:13.185 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:12:12 smithi134 bash[44248]: debug 2024-07-15T07:12:12.908+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6830 osd.3 since back 2024-07-15T07:11:44.099611+0000 front 2024-07-15T07:11:44.099482+0000 (oldest deadline 2024-07-15T07:12:09.999265+0000) 2024-07-15T07:12:13.841 DEBUG:teuthology.orchestra.run.smithi134:> sudo pkill -f 'journalctl -f -n 0 -u ceph-75ba02bc-4274-11ef-bcad-c7b262605968@osd.6.service' 2024-07-15T07:12:13.869 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-07-15T07:12:13.869 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-07-15T07:12:13.870 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-07-15T07:12:13.870 DEBUG:teuthology.orchestra.run.smithi134:> sudo systemctl stop ceph-75ba02bc-4274-11ef-bcad-c7b262605968@osd.7 2024-07-15T07:12:14.308 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:12:13 smithi134 bash[44248]: debug 2024-07-15T07:12:13.892+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:19.694655+0000 front 2024-07-15T07:11:19.695037+0000 (oldest deadline 2024-07-15T07:11:43.794595+0000) 2024-07-15T07:12:14.308 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:12:13 smithi134 bash[44248]: debug 2024-07-15T07:12:13.892+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-07-15T07:11:27.896052+0000 front 2024-07-15T07:11:27.896379+0000 (oldest deadline 2024-07-15T07:11:49.595985+0000) 2024-07-15T07:12:14.308 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:12:13 smithi134 bash[44248]: debug 2024-07-15T07:12:13.892+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-07-15T07:11:34.198177+0000 front 2024-07-15T07:11:34.198069+0000 (oldest deadline 2024-07-15T07:11:58.297764+0000) 2024-07-15T07:12:14.308 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:12:13 smithi134 bash[44248]: debug 2024-07-15T07:12:13.892+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6830 osd.3 since back 2024-07-15T07:11:44.099611+0000 front 2024-07-15T07:11:44.099482+0000 (oldest deadline 2024-07-15T07:12:09.999265+0000) 2024-07-15T07:12:14.309 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:12:13 smithi134 systemd[1]: Stopping Ceph osd.7 for 75ba02bc-4274-11ef-bcad-c7b262605968... 2024-07-15T07:12:14.309 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:12:14 smithi134 bash[44248]: debug 2024-07-15T07:12:14.056+0000 7f78898b9640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-07-15T07:12:14.309 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:12:14 smithi134 bash[44248]: debug 2024-07-15T07:12:14.056+0000 7f78898b9640 -1 osd.7 114 *** Got signal Terminated *** 2024-07-15T07:12:14.309 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:12:14 smithi134 bash[44248]: debug 2024-07-15T07:12:14.056+0000 7f78898b9640 -1 osd.7 114 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-07-15T07:12:15.308 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:12:14 smithi134 bash[44248]: debug 2024-07-15T07:12:14.860+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:19.694655+0000 front 2024-07-15T07:11:19.695037+0000 (oldest deadline 2024-07-15T07:11:43.794595+0000) 2024-07-15T07:12:15.308 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:12:14 smithi134 bash[44248]: debug 2024-07-15T07:12:14.860+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-07-15T07:11:27.896052+0000 front 2024-07-15T07:11:27.896379+0000 (oldest deadline 2024-07-15T07:11:49.595985+0000) 2024-07-15T07:12:15.308 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:12:14 smithi134 bash[44248]: debug 2024-07-15T07:12:14.860+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-07-15T07:11:34.198177+0000 front 2024-07-15T07:11:34.198069+0000 (oldest deadline 2024-07-15T07:11:58.297764+0000) 2024-07-15T07:12:15.308 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:12:14 smithi134 bash[44248]: debug 2024-07-15T07:12:14.860+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6830 osd.3 since back 2024-07-15T07:11:44.099611+0000 front 2024-07-15T07:11:44.099482+0000 (oldest deadline 2024-07-15T07:12:09.999265+0000) 2024-07-15T07:12:16.308 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:12:15 smithi134 bash[44248]: debug 2024-07-15T07:12:15.860+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:19.694655+0000 front 2024-07-15T07:11:19.695037+0000 (oldest deadline 2024-07-15T07:11:43.794595+0000) 2024-07-15T07:12:16.308 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:12:15 smithi134 bash[44248]: debug 2024-07-15T07:12:15.860+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-07-15T07:11:27.896052+0000 front 2024-07-15T07:11:27.896379+0000 (oldest deadline 2024-07-15T07:11:49.595985+0000) 2024-07-15T07:12:16.308 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:12:15 smithi134 bash[44248]: debug 2024-07-15T07:12:15.860+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-07-15T07:11:34.198177+0000 front 2024-07-15T07:11:34.198069+0000 (oldest deadline 2024-07-15T07:11:58.297764+0000) 2024-07-15T07:12:16.308 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:12:15 smithi134 bash[44248]: debug 2024-07-15T07:12:15.860+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6830 osd.3 since back 2024-07-15T07:11:44.099611+0000 front 2024-07-15T07:11:44.099482+0000 (oldest deadline 2024-07-15T07:12:09.999265+0000) 2024-07-15T07:12:17.308 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:12:16 smithi134 bash[44248]: debug 2024-07-15T07:12:16.908+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:19.694655+0000 front 2024-07-15T07:11:19.695037+0000 (oldest deadline 2024-07-15T07:11:43.794595+0000) 2024-07-15T07:12:17.308 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:12:16 smithi134 bash[44248]: debug 2024-07-15T07:12:16.908+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-07-15T07:11:27.896052+0000 front 2024-07-15T07:11:27.896379+0000 (oldest deadline 2024-07-15T07:11:49.595985+0000) 2024-07-15T07:12:17.308 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:12:16 smithi134 bash[44248]: debug 2024-07-15T07:12:16.908+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-07-15T07:11:34.198177+0000 front 2024-07-15T07:11:34.198069+0000 (oldest deadline 2024-07-15T07:11:58.297764+0000) 2024-07-15T07:12:17.308 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:12:16 smithi134 bash[44248]: debug 2024-07-15T07:12:16.908+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6830 osd.3 since back 2024-07-15T07:11:44.099611+0000 front 2024-07-15T07:11:44.099482+0000 (oldest deadline 2024-07-15T07:12:09.999265+0000) 2024-07-15T07:12:18.308 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:12:17 smithi134 bash[44248]: debug 2024-07-15T07:12:17.892+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:19.694655+0000 front 2024-07-15T07:11:19.695037+0000 (oldest deadline 2024-07-15T07:11:43.794595+0000) 2024-07-15T07:12:18.308 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:12:17 smithi134 bash[44248]: debug 2024-07-15T07:12:17.896+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-07-15T07:11:27.896052+0000 front 2024-07-15T07:11:27.896379+0000 (oldest deadline 2024-07-15T07:11:49.595985+0000) 2024-07-15T07:12:18.308 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:12:17 smithi134 bash[44248]: debug 2024-07-15T07:12:17.896+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-07-15T07:11:34.198177+0000 front 2024-07-15T07:11:34.198069+0000 (oldest deadline 2024-07-15T07:11:58.297764+0000) 2024-07-15T07:12:18.308 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:12:17 smithi134 bash[44248]: debug 2024-07-15T07:12:17.896+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6830 osd.3 since back 2024-07-15T07:11:44.099611+0000 front 2024-07-15T07:11:44.099482+0000 (oldest deadline 2024-07-15T07:12:09.999265+0000) 2024-07-15T07:12:18.308 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:12:17 smithi134 bash[44248]: debug 2024-07-15T07:12:17.896+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.134:6804 osd.4 since back 2024-07-15T07:11:51.700673+0000 front 2024-07-15T07:11:51.700750+0000 (oldest deadline 2024-07-15T07:12:17.000394+0000) 2024-07-15T07:12:19.308 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:12:18 smithi134 bash[44248]: debug 2024-07-15T07:12:18.856+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-07-15T07:11:19.694655+0000 front 2024-07-15T07:11:19.695037+0000 (oldest deadline 2024-07-15T07:11:43.794595+0000) 2024-07-15T07:12:19.308 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:12:18 smithi134 bash[44248]: debug 2024-07-15T07:12:18.856+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-07-15T07:11:27.896052+0000 front 2024-07-15T07:11:27.896379+0000 (oldest deadline 2024-07-15T07:11:49.595985+0000) 2024-07-15T07:12:19.308 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:12:18 smithi134 bash[44248]: debug 2024-07-15T07:12:18.856+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-07-15T07:11:34.198177+0000 front 2024-07-15T07:11:34.198069+0000 (oldest deadline 2024-07-15T07:11:58.297764+0000) 2024-07-15T07:12:19.308 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:12:18 smithi134 bash[44248]: debug 2024-07-15T07:12:18.856+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.89:6830 osd.3 since back 2024-07-15T07:11:44.099611+0000 front 2024-07-15T07:11:44.099482+0000 (oldest deadline 2024-07-15T07:12:09.999265+0000) 2024-07-15T07:12:19.308 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:12:18 smithi134 bash[44248]: debug 2024-07-15T07:12:18.856+0000 7f7884ecf640 -1 osd.7 114 heartbeat_check: no reply from 172.21.15.134:6804 osd.4 since back 2024-07-15T07:11:51.700673+0000 front 2024-07-15T07:11:51.700750+0000 (oldest deadline 2024-07-15T07:12:17.000394+0000) 2024-07-15T07:12:20.393 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:12:20 smithi134 bash[80017]: ceph-75ba02bc-4274-11ef-bcad-c7b262605968-osd-7 2024-07-15T07:12:20.393 INFO:journalctl@ceph.osd.7.smithi134.stdout:Jul 15 07:12:20 smithi134 bash[80080]: Error response from daemon: No such container: ceph-75ba02bc-4274-11ef-bcad-c7b262605968-osd-7 2024-07-15T07:12:20.621 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 07:12:20 smithi089 bash[72306]: ts=2024-07-15T07:12:20.177Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi089.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.89:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi134.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.134:8443: connect: connection refused" 2024-07-15T07:12:20.622 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 07:12:20 smithi089 bash[72306]: ts=2024-07-15T07:12:20.178Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi089.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.89:8443: connect: connection refused" 2024-07-15T07:12:20.622 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 07:12:20 smithi089 bash[72306]: ts=2024-07-15T07:12:20.180Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi134.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.134:8443: connect: connection refused" 2024-07-15T07:12:22.216 DEBUG:teuthology.orchestra.run.smithi134:> sudo pkill -f 'journalctl -f -n 0 -u ceph-75ba02bc-4274-11ef-bcad-c7b262605968@osd.7.service' 2024-07-15T07:12:22.244 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-07-15T07:12:22.245 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-07-15T07:12:22.245 INFO:tasks.cephadm.prometheus.a:Stopping prometheus.a... 2024-07-15T07:12:22.245 DEBUG:teuthology.orchestra.run.smithi134:> sudo systemctl stop ceph-75ba02bc-4274-11ef-bcad-c7b262605968@prometheus.a 2024-07-15T07:12:22.464 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:12:22 smithi134 systemd[1]: Stopping Ceph prometheus.a for 75ba02bc-4274-11ef-bcad-c7b262605968... 2024-07-15T07:12:22.464 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:12:22 smithi134 bash[60729]: ts=2024-07-15T07:12:22.423Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-07-15T07:12:22.464 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:12:22 smithi134 bash[60729]: ts=2024-07-15T07:12:22.423Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-07-15T07:12:22.465 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:12:22 smithi134 bash[60729]: ts=2024-07-15T07:12:22.423Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-07-15T07:12:22.465 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:12:22 smithi134 bash[60729]: ts=2024-07-15T07:12:22.423Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-07-15T07:12:22.465 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:12:22 smithi134 bash[60729]: ts=2024-07-15T07:12:22.423Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-07-15T07:12:22.465 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:12:22 smithi134 bash[60729]: ts=2024-07-15T07:12:22.423Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-07-15T07:12:22.465 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:12:22 smithi134 bash[60729]: ts=2024-07-15T07:12:22.423Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-07-15T07:12:22.465 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:12:22 smithi134 bash[60729]: ts=2024-07-15T07:12:22.423Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-07-15T07:12:22.465 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:12:22 smithi134 bash[60729]: ts=2024-07-15T07:12:22.423Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-07-15T07:12:22.465 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:12:22 smithi134 bash[60729]: ts=2024-07-15T07:12:22.423Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-07-15T07:12:22.466 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:12:22 smithi134 bash[60729]: ts=2024-07-15T07:12:22.423Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-07-15T07:12:22.466 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:12:22 smithi134 bash[60729]: ts=2024-07-15T07:12:22.423Z caller=main.go:1147 level=info msg="See you next time!" 2024-07-15T07:12:23.138 DEBUG:teuthology.orchestra.run.smithi134:> sudo pkill -f 'journalctl -f -n 0 -u ceph-75ba02bc-4274-11ef-bcad-c7b262605968@prometheus.a.service' 2024-07-15T07:12:23.146 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:12:22 smithi134 bash[80210]: ceph-75ba02bc-4274-11ef-bcad-c7b262605968-prometheus-a 2024-07-15T07:12:23.147 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:12:23 smithi134 bash[80270]: Error response from daemon: No such container: ceph-75ba02bc-4274-11ef-bcad-c7b262605968-prometheus-a 2024-07-15T07:12:23.147 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:12:23 smithi134 systemd[1]: ceph-75ba02bc-4274-11ef-bcad-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-07-15T07:12:23.147 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Jul 15 07:12:23 smithi134 systemd[1]: Stopped Ceph prometheus.a for 75ba02bc-4274-11ef-bcad-c7b262605968. 2024-07-15T07:12:23.160 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-07-15T07:12:23.161 INFO:tasks.cephadm.prometheus.a:Stopped prometheus.a 2024-07-15T07:12:23.161 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 --force --keep-logs 2024-07-15T07:12:28.168 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 07:12:28 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:12:28.168 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 07:12:28 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:12:28.433 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 07:12:28 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:12:28.433 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 07:12:28 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:12:28.813 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 07:12:28 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:12:28.813 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 07:12:28 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:12:29.109 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 07:12:28 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:12:29.109 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 07:12:29 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:12:29.109 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 07:12:28 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:12:29.109 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 07:12:29 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:12:29.371 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 07:12:29 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:12:29.372 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 07:12:29 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:12:29.720 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 07:12:29 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:12:29.721 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 07:12:29 smithi089 systemd[1]: Stopping Ceph node-exporter.a for 75ba02bc-4274-11ef-bcad-c7b262605968... 2024-07-15T07:12:29.721 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 07:12:29 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:12:30.504 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 07:12:30 smithi089 bash[72306]: ts=2024-07-15T07:12:30.178Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi089.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.89:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi134.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.134:8443: connect: connection refused" 2024-07-15T07:12:30.504 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 07:12:30 smithi089 bash[72306]: ts=2024-07-15T07:12:30.179Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi089.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.89:8443: connect: connection refused" 2024-07-15T07:12:30.504 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 07:12:30 smithi089 bash[72306]: ts=2024-07-15T07:12:30.180Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi134.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.134:8443: connect: connection refused" 2024-07-15T07:12:30.505 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 07:12:30 smithi089 bash[119693]: ceph-75ba02bc-4274-11ef-bcad-c7b262605968-node-exporter-a 2024-07-15T07:12:30.505 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 07:12:30 smithi089 systemd[1]: ceph-75ba02bc-4274-11ef-bcad-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-07-15T07:12:30.505 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 07:12:30 smithi089 bash[119753]: Error response from daemon: No such container: ceph-75ba02bc-4274-11ef-bcad-c7b262605968-node-exporter-a 2024-07-15T07:12:30.505 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 07:12:30 smithi089 systemd[1]: ceph-75ba02bc-4274-11ef-bcad-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-07-15T07:12:30.505 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 07:12:30 smithi089 systemd[1]: Stopped Ceph node-exporter.a for 75ba02bc-4274-11ef-bcad-c7b262605968. 2024-07-15T07:12:30.764 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jul 15 07:12:30 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:12:30.765 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 07:12:30 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:12:30.766 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 07:12:30 smithi089 systemd[1]: Stopping Ceph alertmanager.a for 75ba02bc-4274-11ef-bcad-c7b262605968... 2024-07-15T07:12:30.766 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 07:12:30 smithi089 bash[72306]: ts=2024-07-15T07:12:30.721Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-07-15T07:12:31.554 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 07:12:31 smithi089 bash[119816]: ceph-75ba02bc-4274-11ef-bcad-c7b262605968-alertmanager-a 2024-07-15T07:12:31.555 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 07:12:31 smithi089 bash[119876]: Error response from daemon: No such container: ceph-75ba02bc-4274-11ef-bcad-c7b262605968-alertmanager-a 2024-07-15T07:12:31.555 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 07:12:31 smithi089 systemd[1]: ceph-75ba02bc-4274-11ef-bcad-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-07-15T07:12:31.555 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 07:12:31 smithi089 systemd[1]: Stopped Ceph alertmanager.a for 75ba02bc-4274-11ef-bcad-c7b262605968. 2024-07-15T07:12:31.871 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jul 15 07:12:31 smithi089 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:12:34.327 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 --force --keep-logs 2024-07-15T07:12:39.346 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 07:12:39 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:12:39.346 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 07:12:39 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:12:39.599 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 07:12:39 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:12:39.599 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 07:12:39 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:12:39.957 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 07:12:39 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:12:39.957 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 07:12:39 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:12:40.267 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 07:12:39 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:12:40.267 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 07:12:40 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:12:40.267 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 07:12:39 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:12:40.267 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 07:12:40 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:12:40.553 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 07:12:40 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:12:40.553 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 07:12:40 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:12:40.807 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 07:12:40 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:12:40.808 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 07:12:40 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:12:40.808 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 07:12:40 smithi134 systemd[1]: Stopping Ceph node-exporter.b for 75ba02bc-4274-11ef-bcad-c7b262605968... 2024-07-15T07:12:41.732 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 07:12:41 smithi134 bash[80535]: ceph-75ba02bc-4274-11ef-bcad-c7b262605968-node-exporter-b 2024-07-15T07:12:41.732 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 07:12:41 smithi134 systemd[1]: ceph-75ba02bc-4274-11ef-bcad-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-07-15T07:12:41.732 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 07:12:41 smithi134 bash[80597]: Error response from daemon: No such container: ceph-75ba02bc-4274-11ef-bcad-c7b262605968-node-exporter-b 2024-07-15T07:12:41.732 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 07:12:41 smithi134 systemd[1]: ceph-75ba02bc-4274-11ef-bcad-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-07-15T07:12:41.732 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 07:12:41 smithi134 systemd[1]: Stopped Ceph node-exporter.b for 75ba02bc-4274-11ef-bcad-c7b262605968. 2024-07-15T07:12:41.984 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Jul 15 07:12:41 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:12:41.984 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 07:12:41 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:12:41.984 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 07:12:41 smithi134 systemd[1]: Stopping Ceph grafana.a for 75ba02bc-4274-11ef-bcad-c7b262605968... 2024-07-15T07:12:42.738 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 07:12:42 smithi134 bash[80661]: ceph-75ba02bc-4274-11ef-bcad-c7b262605968-grafana-a 2024-07-15T07:12:42.738 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 07:12:42 smithi134 systemd[1]: ceph-75ba02bc-4274-11ef-bcad-c7b262605968@grafana.a.service: Main process exited, code=exited, status=143/n/a 2024-07-15T07:12:42.738 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 07:12:42 smithi134 bash[80721]: Error response from daemon: No such container: ceph-75ba02bc-4274-11ef-bcad-c7b262605968-grafana-a 2024-07-15T07:12:42.739 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 07:12:42 smithi134 systemd[1]: ceph-75ba02bc-4274-11ef-bcad-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-07-15T07:12:42.739 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 07:12:42 smithi134 systemd[1]: Stopped Ceph grafana.a for 75ba02bc-4274-11ef-bcad-c7b262605968. 2024-07-15T07:12:43.058 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Jul 15 07:12:42 smithi134 systemd[1]: /etc/systemd/system/ceph-75ba02bc-4274-11ef-bcad-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-15T07:12:51.345 DEBUG:teuthology.orchestra.run.smithi089:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-07-15T07:12:51.359 DEBUG:teuthology.orchestra.run.smithi134:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-07-15T07:12:51.371 INFO:tasks.cephadm:Archiving crash dumps... 2024-07-15T07:12:51.373 DEBUG:teuthology.misc:Transferring archived files from smithi089:/var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/crash to /home/teuthworker/archive/teuthology-2024-07-10_01:16:02-upgrade:quincy-x-reef-distro-default-smithi/7795172/remote/smithi089/crash 2024-07-15T07:12:51.374 DEBUG:teuthology.orchestra.run.smithi089:> sudo tar c -f - -C /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/crash -- . 2024-07-15T07:12:51.415 INFO:teuthology.orchestra.run.smithi089.stderr:tar: /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/crash: Cannot open: No such file or directory 2024-07-15T07:12:51.415 INFO:teuthology.orchestra.run.smithi089.stderr:tar: Error is not recoverable: exiting now 2024-07-15T07:12:51.417 DEBUG:teuthology.misc:Transferring archived files from smithi134:/var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/crash to /home/teuthworker/archive/teuthology-2024-07-10_01:16:02-upgrade:quincy-x-reef-distro-default-smithi/7795172/remote/smithi134/crash 2024-07-15T07:12:51.418 DEBUG:teuthology.orchestra.run.smithi134:> sudo tar c -f - -C /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/crash -- . 2024-07-15T07:12:51.429 INFO:teuthology.orchestra.run.smithi134.stderr:tar: /var/lib/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/crash: Cannot open: No such file or directory 2024-07-15T07:12:51.429 INFO:teuthology.orchestra.run.smithi134.stderr:tar: Error is not recoverable: exiting now 2024-07-15T07:12:51.430 INFO:tasks.cephadm:Checking cluster log for badness... 2024-07-15T07:12:51.430 DEBUG:teuthology.orchestra.run.smithi089:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-07-15T07:12:51.471 INFO:teuthology.orchestra.run.smithi089.stderr:grep: /var/log/ceph/75ba02bc-4274-11ef-bcad-c7b262605968/ceph.log: No such file or directory 2024-07-15T07:12:51.473 INFO:tasks.cephadm:Compressing logs... 2024-07-15T07:12:51.473 DEBUG:teuthology.orchestra.run.smithi089:> sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-07-15T07:12:51.517 DEBUG:teuthology.orchestra.run.smithi134:> sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-07-15T07:12:51.527 INFO:teuthology.orchestra.run.smithi089.stderr:find: '/var/log/rbd-target-api': No such file or directory 2024-07-15T07:12:51.529 INFO:teuthology.orchestra.run.smithi134.stderr:find: '/var/log/rbd-target-api': No such file or directory 2024-07-15T07:12:56.806 INFO:tasks.cephadm:Archiving logs... 2024-07-15T07:12:56.806 DEBUG:teuthology.misc:Transferring archived files from smithi089:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-07-10_01:16:02-upgrade:quincy-x-reef-distro-default-smithi/7795172/remote/smithi089/log 2024-07-15T07:12:56.807 DEBUG:teuthology.orchestra.run.smithi089:> sudo tar c -f - -C /var/log/ceph -- . 2024-07-15T07:12:57.410 DEBUG:teuthology.misc:Transferring archived files from smithi134:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-07-10_01:16:02-upgrade:quincy-x-reef-distro-default-smithi/7795172/remote/smithi134/log 2024-07-15T07:12:57.411 DEBUG:teuthology.orchestra.run.smithi134:> sudo tar c -f - -C /var/log/ceph -- . 2024-07-15T07:12:57.786 INFO:tasks.cephadm:Removing cluster... 2024-07-15T07:12:57.787 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 --force 2024-07-15T07:12:59.313 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 75ba02bc-4274-11ef-bcad-c7b262605968 --force 2024-07-15T07:13:00.826 INFO:tasks.cephadm:Removing cephadm ... 2024-07-15T07:13:00.826 DEBUG:teuthology.orchestra.run.smithi089:> rm -rf /home/ubuntu/cephtest/cephadm 2024-07-15T07:13:00.832 DEBUG:teuthology.orchestra.run.smithi134:> rm -rf /home/ubuntu/cephtest/cephadm 2024-07-15T07:13:00.838 INFO:tasks.cephadm:Teardown complete 2024-07-15T07:13:00.838 DEBUG:teuthology.run_tasks:Unwinding manager install 2024-07-15T07:13:00.848 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2024-07-15T07:13:00.849 DEBUG:teuthology.orchestra.run.smithi089:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-07-15T07:13:00.877 DEBUG:teuthology.orchestra.run.smithi134:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-07-15T07:13:00.941 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-07-15T07:13:00.950 INFO:teuthology.task.clock:Checking final clock skew... 2024-07-15T07:13:00.950 DEBUG:teuthology.orchestra.run.smithi089:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-07-15T07:13:00.953 DEBUG:teuthology.orchestra.run.smithi134:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-07-15T07:13:00.962 INFO:teuthology.orchestra.run.smithi089.stdout: remote refid st t when poll reach delay offset jitter 2024-07-15T07:13:00.962 INFO:teuthology.orchestra.run.smithi089.stdout:============================================================================== 2024-07-15T07:13:00.962 INFO:teuthology.orchestra.run.smithi089.stdout:*hv01.front.sepi 67.205.162.81 3 u 47 64 377 0.086 -4.569 0.693 2024-07-15T07:13:00.962 INFO:teuthology.orchestra.run.smithi089.stdout:+hv02.front.sepi 63.231.80.2 3 u 45 64 377 0.097 -3.728 0.604 2024-07-15T07:13:00.963 INFO:teuthology.orchestra.run.smithi089.stdout:+hv03.front.sepi 74.6.168.72 3 u 38 64 377 0.093 -4.474 0.632 2024-07-15T07:13:00.963 INFO:teuthology.orchestra.run.smithi089.stdout: hv04.front.sepi .INIT. 16 u - 1024 0 0.000 +0.000 0.000 2024-07-15T07:13:00.964 INFO:teuthology.orchestra.run.smithi134.stdout: remote refid st t when poll reach delay offset jitter 2024-07-15T07:13:00.965 INFO:teuthology.orchestra.run.smithi134.stdout:============================================================================== 2024-07-15T07:13:00.965 INFO:teuthology.orchestra.run.smithi134.stdout:*hv01.front.sepi 67.205.162.81 3 u 47 64 377 0.086 -4.738 0.812 2024-07-15T07:13:00.965 INFO:teuthology.orchestra.run.smithi134.stdout:+hv02.front.sepi 63.231.80.2 3 u 39 64 377 0.091 -2.434 1.497 2024-07-15T07:13:00.965 INFO:teuthology.orchestra.run.smithi134.stdout:+hv03.front.sepi 74.6.168.72 3 u 32 64 377 0.092 -3.231 1.382 2024-07-15T07:13:00.965 INFO:teuthology.orchestra.run.smithi134.stdout: hv04.front.sepi .INIT. 16 u - 1024 0 0.000 +0.000 0.000 2024-07-15T07:13:00.966 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-07-15T07:13:00.975 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-07-15T07:13:00.975 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-07-15T07:13:00.984 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-07-15T07:13:00.993 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-07-15T07:13:01.001 INFO:teuthology.task.internal:Duration was 2766.784727 seconds 2024-07-15T07:13:01.002 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-07-15T07:13:01.010 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-07-15T07:13:01.011 DEBUG:teuthology.orchestra.run.smithi089:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-07-15T07:13:01.013 DEBUG:teuthology.orchestra.run.smithi134:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-07-15T07:13:01.047 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-07-15T07:13:01.047 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi089.front.sepia.ceph.com 2024-07-15T07:13:01.048 DEBUG:teuthology.orchestra.run.smithi089:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-07-15T07:13:01.096 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi134.front.sepia.ceph.com 2024-07-15T07:13:01.097 DEBUG:teuthology.orchestra.run.smithi134:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-07-15T07:13:01.106 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-07-15T07:13:01.106 DEBUG:teuthology.orchestra.run.smithi089:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-07-15T07:13:01.141 DEBUG:teuthology.orchestra.run.smithi134:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-07-15T07:13:01.205 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-07-15T07:13:01.206 DEBUG:teuthology.orchestra.run.smithi089:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-07-15T07:13:01.208 DEBUG:teuthology.orchestra.run.smithi134:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-07-15T07:13:01.381 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-07-15T07:13:01.392 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-07-15T07:13:01.392 DEBUG:teuthology.orchestra.run.smithi089:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-07-15T07:13:01.404 DEBUG:teuthology.orchestra.run.smithi134:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-07-15T07:13:01.417 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-07-15T07:13:01.428 DEBUG:teuthology.orchestra.run.smithi089:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-07-15T07:13:01.449 DEBUG:teuthology.orchestra.run.smithi134:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-07-15T07:13:01.456 INFO:teuthology.orchestra.run.smithi089.stdout:kernel.core_pattern = core 2024-07-15T07:13:01.469 INFO:teuthology.orchestra.run.smithi134.stdout:kernel.core_pattern = core 2024-07-15T07:13:01.494 DEBUG:teuthology.orchestra.run.smithi089:> test -e /home/ubuntu/cephtest/archive/coredump 2024-07-15T07:13:01.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-07-15T07:13:01.515 DEBUG:teuthology.orchestra.run.smithi134:> test -e /home/ubuntu/cephtest/archive/coredump 2024-07-15T07:13:01.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-07-15T07:13:01.543 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-07-15T07:13:01.553 INFO:teuthology.task.internal:Transferring archived files... 2024-07-15T07:13:01.554 DEBUG:teuthology.misc:Transferring archived files from smithi089:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-07-10_01:16:02-upgrade:quincy-x-reef-distro-default-smithi/7795172/remote/smithi089 2024-07-15T07:13:01.554 DEBUG:teuthology.orchestra.run.smithi089:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-07-15T07:13:01.589 DEBUG:teuthology.misc:Transferring archived files from smithi134:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-07-10_01:16:02-upgrade:quincy-x-reef-distro-default-smithi/7795172/remote/smithi134 2024-07-15T07:13:01.589 DEBUG:teuthology.orchestra.run.smithi134:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-07-15T07:13:01.614 INFO:teuthology.task.internal:Removing archive directory... 2024-07-15T07:13:01.614 DEBUG:teuthology.orchestra.run.smithi089:> rm -rf -- /home/ubuntu/cephtest/archive 2024-07-15T07:13:01.625 DEBUG:teuthology.orchestra.run.smithi134:> rm -rf -- /home/ubuntu/cephtest/archive 2024-07-15T07:13:01.660 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-07-15T07:13:01.671 INFO:teuthology.task.internal:Not uploading archives. 2024-07-15T07:13:01.672 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-07-15T07:13:01.681 INFO:teuthology.task.internal:Tidying up after the test... 2024-07-15T07:13:01.681 DEBUG:teuthology.orchestra.run.smithi089:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-07-15T07:13:01.684 DEBUG:teuthology.orchestra.run.smithi134:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-07-15T07:13:01.688 INFO:teuthology.orchestra.run.smithi089.stdout: 658615 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Jul 15 07:13 /home/ubuntu/cephtest 2024-07-15T07:13:01.707 INFO:teuthology.orchestra.run.smithi134.stdout: 658623 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Jul 15 07:13 /home/ubuntu/cephtest 2024-07-15T07:13:01.709 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-07-15T07:13:01.719 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-07-15T07:13:01.741 INFO:teuthology.run:Summary data: description: upgrade:quincy-x/stress-split/{0-distro/ubuntu_22.04 0-roles 1-start 2-first-half-tasks/rbd-cls 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/classic} duration: 2766.784726858139 failure_reason: timeout expired in wait_until_healthy flavor: default owner: scheduled_teuthology@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=76429c7c47aa429a8711dcf795f3cb96 status: fail success: false 2024-07-15T07:13:01.741 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-07-15T07:13:01.832 INFO:teuthology.run:FAIL